Skip to content

Commit

Permalink
Moved JSON summary prompt to mention score is an integer (#748)
Browse files Browse the repository at this point in the history
Co-authored-by: Andrew White <white.d.andrew@gmail.com>
  • Loading branch information
jamesbraza and whitead authored Dec 4, 2024
1 parent 41cfc25 commit c36903a
Show file tree
Hide file tree
Showing 4 changed files with 2,523 additions and 2,511 deletions.
2 changes: 1 addition & 1 deletion paperqa/prompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
"relevance_score": "..."
}}
where `summary` is relevant information from text - {summary_length} words and `relevance_score` is the relevance of `summary` to answer question (out of 10).
where `summary` is relevant information from the text - {summary_length} words. `relevance_score` is an integer 1-10 for the relevance of `summary` to the question.
""" # noqa: E501

env_system_prompt = (
Expand Down
17 changes: 12 additions & 5 deletions paperqa/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,12 +130,19 @@ def strip_citations(text: str) -> str:


def extract_score(text: str) -> int:
# check for N/A
"""
Extract an integer score from the text in 0 to 10.
Note: score is 1-10, and we use 0 as a sentinel for not applicable.
"""
# Check for N/A, not applicable, not relevant.
# Don't check for NA, as there can be genes containing "NA"
last_line = text.split("\n")[-1]
if "N/A" in last_line or "n/a" in last_line or "NA" in last_line:
return 0
# check for not applicable, not relevant in summary
if "not applicable" in text.lower() or "not relevant" in text.lower():
if (
"n/a" in last_line.lower()
or "not applicable" in text.lower()
or "not relevant" in text.lower()
):
return 0

score = re.search(r"[sS]core[:is\s]+([0-9]+)", text)
Expand Down
Loading

0 comments on commit c36903a

Please sign in to comment.