Python Multiprocessing – ZeroMQ vs Queue

As a quick follow up to my previous post, here’s a look at the performance of passing messages between two python processes using the Queue class vs using 0mq push / pull connections.  As a quick test, we will pass 10 million messages between two processes, first using Queue, then using 0mq.

Multiprocessing test with Queue

import sys
import time
from  multiprocessing import Process, Queue

def worker(q):
    for task_nbr in range(10000000):
        message = q.get()
    sys.exit(1)
 
def main():
    send_q = Queue()
    Process(target=worker, args=(send_q,)).start()
    for num in range(10000000):
        send_q.put("MESSAGE")

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    duration = end_time - start_time
    msg_per_sec = 10000000 / duration

    print "Duration: %s" % duration
    print "Messages Per Second: %s" % msg_per_sec
 

Multiprocessing test with 0mq

import sys
import zmq
from  multiprocessing import Process
import time

def worker():
    context = zmq.Context()
    work_receiver = context.socket(zmq.PULL)
    work_receiver.connect("tcp://127.0.0.1:5557")

    for task_nbr in range(10000000):
        message = work_receiver.recv()

    sys.exit(1)

def main():
    Process(target=worker, args=()).start()
    context = zmq.Context()
    ventilator_send = context.socket(zmq.PUSH)
    ventilator_send.bind("tcp://127.0.0.1:5557")
    for num in range(10000000):
        ventilator_send.send("MESSAGE")

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    duration = end_time - start_time
    msg_per_sec = 10000000 / duration

    print "Duration: %s" % duration
    print "Messages Per Second: %s" % msg_per_sec

Queue Results

python2 ./multiproc_with_queue.py
Duration: 164.182257891
Messages Per Second: 60907.9210414

0mq Results

python2 ./multiproc_with_zeromq.py
Duration: 23.3490710258
Messages Per Second: 428282.563744

The numbers speak for themselves.