Mockito.when won't work inside Test method body

I'm trying to test that Invoice class I made. Works so far, but I have a problem inside of the falsePayCallKeepsPaidAtFalse method. The Mockito.when method does not have any impact on the actual return value. Mockitor.when does only work inside of the setUp() method so far.

What I have tried so far: Remove the Mockitor.when from the setUp() method and have it invoked in each test method. Put the return value to false inside of the setUp() method (That will have an impact). Checked if the problem lies in the actual Invoice class, it does not.

public class InvoiceTest {
    @Mock
    private OrderManager orderManager;
    private InvoiceManager invoiceManager;

    private Invoice invoice;
    private ArrayList<Integer> orderIds;

    @BeforeEach
    public void setUp() {
        orderManager = mock(OrderManager.class);
        invoiceManager = mock(InvoiceManager.class);

        orderIds = new ArrayList<>();
        orderIds.add(2);
        orderIds.add(5);
        invoice = new Invoice(
                null,
                null,
                null,
                null,
                "VAT",
                16,
                false,
                2,
                 orderIds,
                "test",
                LocalDate.now()
        );

        HashMap<Integer, Order> returnOrders = new HashMap();
        returnOrders.put(2, mock(Order.class));
        returnOrders.put(5, mock(Order.class));

        when(orderManager.fetchNOrders(orderIds)).thenReturn(returnOrders);
        when(invoiceManager.setInvoicePaid(invoice, orderIds)).thenReturn(true);

        invoice.pay(invoiceManager, orderManager);
    }
    /**
     * Pay.
     */
    @Test
    public void payCallsSetInvoicePaid() {
        verify(invoiceManager, times(1)).setInvoicePaid(invoice, invoice.getOrders());
    }

    @Test
    public void correctPayCallSetsPaidToTrue() {
        invoiceManager.setInvoicePaid(invoice, orderIds);
        assertThat(invoice.isPaid()).isTrue();
    }

    @Test
    public void falsePayCallKeepsPaidAtFalse() {
        when(invoiceManager.setInvoicePaid(invoice, orderIds)).thenReturn(false);

        invoiceManager.setInvoicePaid(invoice, orderIds);
        assertThat(invoice.isPaid()).isFalse();
    }
    @Test
    public void payAsksForOrderFetch() {
        verify(orderManager, times(1)).fetchNOrders(orderIds);
    }
}