The right way to load bulk data

Question

I am trying to load a huge amount of data in memcachedb. I am executing some queries in a MySQL database and I want to save the results of these queries to memcachedb for faster access.

Currently, I just used simple set commands to store the results in memcachedb, but since there are billions of these results, storing them one at a time in a loop is very inefficient and time consuming. So, I was wondering if there is a better way to load data into memcachedb? As a data import wizard in traditional RDMS

I am using pylibmc to connect to memcachedb.

Answer 1

pylibmc set_multi, :

mc.set_multi({
    'key': 'Hello',
    'another': True,
    #[..]
})

. , , , .

, , . memcache . , noreply, . , , .

:

#!/usr/bin/env python

import socket

data = 'set key_1 0 86400 5\r\nabcde\r\n'
data += 'set key_2 0 86400 5\r\nzxcvb\r\n'

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 11211))
s.sendall(data)
print(s.recv(8192))
s.close()

# Verify if it worked!
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 11211))
s.sendall('get key_1\r\n')
s.sendall('get key_2\r\n')
print(s.recv(8192))
s.close()

:

STORED
STORED

VALUE key_1 0 5
abcde
END
VALUE key_2 0 5
zxcvb
END

set:

set <key> <flags> <exptime> <data_size> [noreply]\r\n
<data>\r\n

, ; :

#!/usr/bin/env python

import socket

def make_set(n, data):
    return 'set key_{} 0 86400 {}\r\n{}\r\n'.format(n, len(data), data)

data = open('/etc/aliases').readlines()
commands = [ make_set(n, d.strip()) for n, d in enumerate(data) if d.strip() != '' ]

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 11211))
s.sendall(''.join(commands))
print(s.recv(65000))

# Verify if it worked!
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('localhost', 11211))
for n in range(0, len(commands)):
    s.sendall('get key_{}\r\n'.format(n))
print(s.recv(65000))
s.close()

MySQL, set SQL-! :

select
    concat('set key_', page_id, ' 0 86400 ', length(page_title), '\r\n', page_title, '\r\n')
    as cmd
from page limit 2;

, , , .