Most efficient way of cleaning up files after a test?

I have written some test cases which test a function I have written. The function is to simply count the number of files in a particular directory. Eventually I will have another function which will behave in a certain way depending how many files are in each directory. In this case I am working with two directories. This is my function:
dir_handler.py

from pathlib import Path

def count_files_in_dir(dirpath):
    assert(dirpath.is_dir())
    file_list = []
    for file in dirpath.iterdir():
        if file.is_file():
            file_list.append(file)
    return len(file_list)  

And here are my test cases:
test_dir_handler.py

from imports import *
import os
from main.dir_handler import count_files_in_dir


class DirHandlerTests(unittest.TestCase):

def test_return_count_of_zero_when_no_file_exists_in_input_dir(self):
    self.assertEqual(0, count_files_in_dir(INPUT_FILE_PATH))

def test_return_count_of_zero_when_no_file_exists_in_output_dir(self):
    self.assertEqual(0, count_files_in_dir(OUTPUT_FILE_PATH))

def test_return_count_of_one_when_one_file_exists_in_input_dir(self):
    with open(str(INPUT_FILE_PATH)+ "/"+"input.csv", "w") as file:
        self.assertEqual(1, count_files_in_dir(INPUT_FILE_PATH))

def test_return_count_of_one_when_one_file_exists_in_output_dir(self):
    with open(str(OUTPUT_FILE_PATH)+ "/"+"output.csv", "w") as file:
        self.assertEqual(1, count_files_in_dir(OUTPUT_FILE_PATH))

def test_return_count_of_one_when_one_file_exists_in_output_dir(self):
    with open(str(OUTPUT_FILE_PATH)+ "/"+"output.csv", "w") as file:
        self.assertEqual(1, count_files_in_dir(OUTPUT_FILE_PATH))

def test_return_count_of_two_when_two_files_exists_in_output_dir(self):
    with open(str(OUTPUT_FILE_PATH)+ "/"+"output.csv", "w") as file:
        with open(str(OUTPUT_FILE_PATH)+ "/"+"output2.csv", "w") as file:
            self.assertEqual(2, count_files_in_dir(OUTPUT_FILE_PATH))

#clearing up testing files at the end of test
def tearDown(self):
    try:
        os.remove(str(INPUT_FILE_PATH)+ "/"+"input.csv")
    except FileNotFoundError as e:
        pass
    try:
        os.remove(str(OUTPUT_FILE_PATH)+ "/"+"output.csv")
    except FileNotFoundError as e:
        pass

    try:
        os.remove(str(OUTPUT_FILE_PATH)+ "/"+"output2.csv")
    except FileNotFoundError as e:
        pass

if __name__ == '__main__':
    unittest.main()

As you can see I am having to remove "input2.csv" and "output2.csv" individually which is not very effecient. Both INPUT_FILE_PATH and OUTPUT_FILE_PATH are under the same directory "files". All tests pass but I would like recommendations on the best way of cleaning INPUT_FILE_PATH and OUTPUT_FILE_PATH directories at the end of my tests. Thank you