83 lines
2.3 KiB
Python
83 lines
2.3 KiB
Python
import threading
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from preshed.bloom import BloomFilter
|
|
from preshed.maps import PreshMap
|
|
|
|
|
|
def run_threaded(chunks, closure):
|
|
orig_interval = sys.getswitchinterval()
|
|
sys.setswitchinterval(.0000001)
|
|
n_threads = len(chunks)
|
|
with ThreadPoolExecutor(max_workers=n_threads) as tpe:
|
|
futures = []
|
|
b = threading.Barrier(n_threads)
|
|
for i, chunk in enumerate(chunks):
|
|
futures.append(tpe.submit(closure, b, chunk))
|
|
[f.result() for f in futures]
|
|
sys.setswitchinterval(orig_interval)
|
|
|
|
|
|
def test_multithreaded_bloom_sharing():
|
|
bf = BloomFilter(size=2**16)
|
|
n_threads = 8
|
|
vals = list(range(0, 10000, 10))
|
|
n_vals = len(vals)
|
|
chunk_size = n_vals//n_threads
|
|
assert chunk_size * n_threads == n_vals
|
|
chunks = []
|
|
for i in range(0, n_vals, chunk_size):
|
|
chunks.append(vals[i: i + chunk_size])
|
|
|
|
def worker(b, chunk):
|
|
b.wait()
|
|
for ii in chunk:
|
|
# exercises __contains__, add, and to_bytes
|
|
# all are supposed to be thread-safe
|
|
assert ii not in bf
|
|
bf.add(ii)
|
|
assert ii in bf
|
|
bf._roundtrip()
|
|
|
|
run_threaded(chunks, worker)
|
|
|
|
|
|
def test_multithreaded_map_sharing():
|
|
h = PreshMap()
|
|
n_threads = 8
|
|
keys = list(range(0, 10000, 10))
|
|
vals = list(range(1, 10000, 10))
|
|
n_vals = len(vals)
|
|
chunk_size = n_vals//n_threads
|
|
assert chunk_size * n_threads == n_vals
|
|
chunks = []
|
|
for i in range(0, n_vals, chunk_size):
|
|
chunks.append(zip(keys[i: i + chunk_size], vals[i: i + chunk_size]))
|
|
assert len(chunks) == n_threads
|
|
|
|
def worker(b, chunk):
|
|
b.wait()
|
|
for k, v in chunk:
|
|
# __getitem__
|
|
assert h[k] is None
|
|
# __setitem__
|
|
h[k] = v
|
|
# __getitem__ again
|
|
assert h[k] == v
|
|
# items()
|
|
for (kk, vv) in h.items():
|
|
# None if another thread removed it
|
|
assert h[kk] in (vv, None)
|
|
# pop
|
|
assert h.pop(k) == v
|
|
assert h[k] is None
|
|
# __delitem__
|
|
h[k] = v
|
|
assert h[k] == v
|
|
del h[k]
|
|
assert h[k] is None
|
|
h[k] = v
|
|
|
|
run_threaded(chunks, worker)
|