How to test swiping between Activities in Espresso?

I am creating a test in Espresso that calls swipeRight() to go to the MainActivity where behavior is set before returning to ConsumerSettingsActivity to test Views. However, the issue is that after calling the swipe, I get a NoActivityResumedException that points to the second tested line with R.id.mode_text.

@RunWith(AndroidJUnit4.class)
public class ConsumerSettingsActivityTest {

private ConsumerSettingsActivity mConsumerSettingsActivity;

@Rule
public ActivityTestRule<ConsumerSettingsActivity> mActivityTestRule =
        new ActivityTestRule<>(ConsumerSettingsActivity.class);

@Before
public void initialize() {
    mConsumerSettingsActivity = mActivityTestRule.getActivity();
}

@Test
public void checkCorrectButtonsAreClickableInPhotoModeThenReturnToMainActivity() {

    // swipe to MainActivity and set behavior
    onView(allOf(withId(R.id.settings_top_bar), isDisplayed())).perform(swipeRight());
    onView(withId(R.id.mode_text)).perform(click());
    onView(withId(R.id.photo_mode)).perform(click());


    // test ratio Views
    onView(withId(R.id.ratio_full)).check(matches(isClickable()));
    onView(withId(R.id.ratio_full)).perform(click());
    onView(withId(R.id.ratio_square)).check(matches(isClickable()));
    onView(withId(R.id.ratio_square)).perform(click()); 
...
}
}