How to Become a Hacker
catb.org4 pointsby foooobaba1 comments
import pickle
import threading
import queue
def load_files_in_parallel(filenames, results_queue):
"""Loads a list of files in parallel with the pickle library and outputs the results to a queue.
Args:
filenames: A list of filenames to load.
results_queue: A queue to hold the results of the file loads.
Returns:
None.
"""
# Create a queue to hold the file handles.
file_handles = queue.Queue()
# Create a thread pool to load the files.
threads = []
for _ in range(len(filenames)):
thread = threading.Thread(target=load_file, args=(file_handles, filenames.pop(), results_queue))
threads.append(thread)
thread.start()
# Wait for all threads to finish.
for thread in threads:
thread.join()
# Return the list of objects loaded from the files.
# No need to return anything since the results are already in the queue.
def load_file(file_handles, filename, results_queue):
"""Loads a file and adds it to the results queue.
Args:
file_handles: A queue to hold the file handles.
filename: The name of the file to load.
results_queue: A queue to hold the results of the file loads.
Returns:
None.
"""
# Open the file.
with open(filename, 'rb') as f:
obj = pickle.load(f)
# Add the object to the results queue.
results_queue.put(obj)
[1] Sandhills: https://en.m.wikipedia.org/wiki/Sandhills_(Nebraska)
[2] Ogallala Aquifer: https://en.m.wikipedia.org/wiki/Ogallala_Aquifer