Seeding WebApplicationFactory For Integration Tests Doesn't Save and Gives a 404

So I'm trying to create a integration test with preseeded data in my dbcontext, but for some reason it's not saving my entry to the database and gives me a 404 not found after I get my response back. Happy to expand or clarify anything if needed.

Cheers

[Fact]
public async Task GetPet_ReturnsResourceWithAccurateFields()
{
    var fakePet = new FakePet { }.Generate();
    fakePet.PetId = 1;

    var appFactory = new WebApplicationFactory<StartupAutomatedTesting>()
        .WithWebHostBuilder(builder =>
        {
            builder
                .ConfigureServices(services =>
                {
                    services.Remove(new ServiceDescriptor(typeof(MyDbContext),
                        typeof(MyDbContext)));
                    services.AddDbContext<MyDbContext>(options =>
                    {
                        options.UseInMemoryDatabase("TestDb");
                    });

                    var serviceProvider = services
                        .AddEntityFrameworkInMemoryDatabase()
                        .BuildServiceProvider();

                    using (var scope = serviceProvider.CreateScope())
                    {
                        var context = scope.ServiceProvider.GetRequiredService<MyDbContext>();
                        context.Database.EnsureCreated();

                        context.Pets.AddRange(fakePet);
                        context.SaveChanges();
                    }
                });
        });

    var testingClient = appFactory.CreateClient();


    var result = await testingClient.GetAsync($"/v1/Pets/{fakePet.PetId}");
    var responseContent = await result.Content.ReadAsStringAsync();
    var response = JsonConvert.DeserializeObject<PetDto>(responseContent);

    // Assert
    response.Name.Should().Be(fakePet.Name);
    response.Age.Should().Be(fakePet.Age);
}