Skip to content

Latest commit

 

History

History
241 lines (237 loc) · 6.49 KB

87.md

File metadata and controls

241 lines (237 loc) · 6.49 KB

Results of GPT-4 on HumanEval/87

Date: 18 July, 2023
Model: gpt-4
Temperature: 0.2
Passing runs: 0
Total runs: 10
Success rate: 0%

Prompt

Tokens: 285

def get_row(lst, x):
    """
    You are given a 2 dimensional data, as a nested lists,
    which is similar to matrix, however, unlike matrices,
    each row may contain a different number of columns.
    Given lst, and integer x, find integers x in the list,
    and return list of tuples, [(x1, y1), (x2, y2) ...] such that
    each tuple is a coordinate - (row, columns), starting with 0.
    Sort coordinates initially by rows in ascending order.
    Also, sort coordinates of the row by columns in descending order.
    
    Examples:
    get_row([
      [1,2,3,4,5,6],
      [1,2,3,4,1,6],
      [1,2,3,4,5,1]
    ], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)]
    get_row([], 1) == []
    get_row([[], [1], [1, 2, 3]], 3) == [(2, 2)]
    """

The above is an incomplete Python code fragment. Return the complete and correct code with no additional text.

Correct solution

def get_row(lst, x):
    """
    You are given a 2 dimensional data, as a nested lists,
    which is similar to matrix, however, unlike matrices,
    each row may contain a different number of columns.
    Given lst, and integer x, find integers x in the list,
    and return list of tuples, [(x1, y1), (x2, y2) ...] such that
    each tuple is a coordinate - (row, columns), starting with 0.
    Sort coordinates initially by rows in ascending order.
    Also, sort coordinates of the row by columns in descending order.
    
    Examples:
    get_row([
      [1,2,3,4,5,6],
      [1,2,3,4,1,6],
      [1,2,3,4,5,1]
    ], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)]
    get_row([], 1) == []
    get_row([[], [1], [1, 2, 3]], 3) == [(2, 2)]
    """coords = [(i, j) for i in range(len(lst)) for j in range(len(lst[i])) if lst[i][j] == x]
    return sorted(sorted(coords, key=lambda x: x[1], reverse=True), key=lambda x: x[0])

Unit tests

def check(candidate):

    # Check some simple cases
    assert candidate([
        [1,2,3,4,5,6],
        [1,2,3,4,1,6],
        [1,2,3,4,5,1]
    ], 1) == [(0, 0), (1, 4), (1, 0), (2, 5), (2, 0)]
    assert candidate([
        [1,2,3,4,5,6],
        [1,2,3,4,5,6],
        [1,2,3,4,5,6],
        [1,2,3,4,5,6],
        [1,2,3,4,5,6],
        [1,2,3,4,5,6]
    ], 2) == [(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1)]
    assert candidate([
        [1,2,3,4,5,6],
        [1,2,3,4,5,6],
        [1,1,3,4,5,6],
        [1,2,1,4,5,6],
        [1,2,3,1,5,6],
        [1,2,3,4,1,6],
        [1,2,3,4,5,1]
    ], 1) == [(0, 0), (1, 0), (2, 1), (2, 0), (3, 2), (3, 0), (4, 3), (4, 0), (5, 4), (5, 0), (6, 5), (6, 0)]
    assert candidate([], 1) == []
    assert candidate([[1]], 2) == []
    assert candidate([[], [1], [1, 2, 3]], 3) == [(2, 2)]

    # Check some edge cases that are easy to work out by hand.
    assert True

Runs

❌ Run 1

Start time: 2023-07-18 18:20:34
Duration: 5.25s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 2

Start time: 2023-07-18 18:20:25
Duration: 8.71s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 3

Start time: 2023-07-18 18:20:20
Duration: 4.72s
Completion tokens: 70
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], x[1]))
    return result

❌ Run 4

Start time: 2023-07-18 18:20:11
Duration: 8.88s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 5

Start time: 2023-07-18 18:20:06
Duration: 5.46s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 6

Start time: 2023-07-18 18:20:00
Duration: 5.49s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 7

Start time: 2023-07-18 18:19:54
Duration: 6.25s
Completion tokens: 70
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], x[1]))
    return result

❌ Run 8

Start time: 2023-07-18 18:19:47
Duration: 6.91s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 9

Start time: 2023-07-18 18:19:41
Duration: 6.02s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result

❌ Run 10

Start time: 2023-07-18 18:19:35
Duration: 5.97s
Completion tokens: 71
Result: Fail

def get_row(lst, x):
    result = []
    for i in range(len(lst)):
        for j in range(len(lst[i])):
            if lst[i][j] == x:
                result.append((i, j))
    result.sort(key=lambda x: (-x[0], -x[1]))
    return result