Skip to content

Latest commit

 

History

History
338 lines (308 loc) · 9.33 KB

91.md

File metadata and controls

338 lines (308 loc) · 9.33 KB
Raw

Results of GPT-3.5 on HumanEval/91

Date: 18 July, 2023
Model: gpt-3.5-turbo
Temperature: 0.2
Passing runs: 0
Total runs: 10
Success rate: 0%

Prompt

Tokens: 133

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """

The above is an incomplete Python code fragment. Return the complete and correct code with no additional text.

Correct solution

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """import re
    sentences = re.split(r'[.?!]\s*', S)
    return sum(sentence[0:2] == 'I ' for sentence in sentences)

Unit tests

def check(candidate):

    # Check some simple cases
    assert candidate("Hello world") == 0, "Test 1"
    assert candidate("Is the sky blue?") == 0, "Test 2"
    assert candidate("I love It !") == 1, "Test 3"
    assert candidate("bIt") == 0, "Test 4"
    assert candidate("I feel good today. I will be productive. will kill It") == 2, "Test 5"
    assert candidate("You and I are going for a walk") == 0, "Test 6"

    # Check some edge cases that are easy to work out by hand.
    assert True, "This prints if this assert fails 2 (also good for debugging!)"

Runs

❌ Run 1

Start time: 2023-07-18 17:14:39
Duration: 6.13s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 2

Start time: 2023-07-18 17:14:32
Duration: 6.2s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 3

Start time: 2023-07-18 17:14:25
Duration: 6.7s
Completion tokens: 167
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split('. ')
    sentences = [sentence.split('? ') for sentence in sentences]
    sentences = [sentence.split('! ') for sublist in sentences for sentence in sublist]
    count = 0
    for sentence in sentences:
        if sentence.startswith('I'):
            count += 1
    return count

❌ Run 4

Start time: 2023-07-18 17:14:19
Duration: 6.08s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 5

Start time: 2023-07-18 17:14:13
Duration: 6.48s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 6

Start time: 2023-07-18 17:14:07
Duration: 6.0s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 7

Start time: 2023-07-18 17:14:02
Duration: 4.82s
Completion tokens: 129
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split('. ')
    sentences = [sentence for sentence in sentences if sentence.startswith('I')]
    return len(sentences)

❌ Run 8

Start time: 2023-07-18 17:13:56
Duration: 5.99s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 9

Start time: 2023-07-18 17:13:50
Duration: 5.86s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count

❌ Run 10

Start time: 2023-07-18 17:13:46
Duration: 4.09s
Completion tokens: 161
Result: Fail

def is_bored(S):
    """
    You'll be given a string of words, and your task is to count the number
    of boredoms. A boredom is a sentence that starts with the word "I".
    Sentences are delimited by '.', '?' or '!'.
   
    For example:
    >>> is_bored("Hello world")
    0
    >>> is_bored("The sky is blue. The sun is shining. I love this weather")
    1
    """
    sentences = S.split(".")
    sentences = [sentence.strip() for sentence in sentences]
    sentences = [sentence for sentence in sentences if sentence]
    
    count = 0
    for sentence in sentences:
        if sentence.startswith("I"):
            count += 1
    
    return count