[python] openTSDB fast R/W test

Write

from telnetlib import Telnet  
import time
import socket
import uuid 
import pickle
import random
import threading

host = "localhost"
port = 4242
sum, n = (0,0)
def gen_uuid():
    list = [ str(uuid.uuid4()) for i in xrange(10000)]
    with open("uuid.txt", "wb") as fp:   #Pickling
        pickle.dump(list, fp)
    print "generate uuid done"
def load_uuid():
    while True:
        with open("uuid.txt", "rb") as f:
            return pickle.load(f)
def load_ip():
    while True:
        with open("ip.txt", "rb") as f:
            return pickle.load(f)
def load_api():
    l = []    
    services = ['107','108','109','129','134','133','134','191','http_src_based_glid_drop','http_blacklist_drop',
                'http_tcp_service_drop','http_service_drop','http_auth_drop','http_port_glid_drop','http_tcp_auth_drop']
    #now = 1563303945
    now = int(time.time())
    ip = load_ip()
    uuid = load_uuid()
    for timestamp in range(now, now-86400*2, -15):
        yield [ "put {uuid}.{service} {timestamp} {val} host=test \n".format(
                    uuid = uuid[i],
                    service = service,
                    timestamp = timestamp,
                    val = timestamp%101,
                ) for i in range(10000)
            for service in services]

#gen_uuid()
def callback():
    print "ok"
def write_db(msg):
    while True:
        try:
            tn = Telnet(host, port)
            for i in msg:
                tn.write(i)
            tn.close()
            break
        except socket.error as e:
            pass
_api = load_api()
n = 0
while True:
    start = time.time()
    tl = []
    for i in xrange(10):
        thr = threading.Thread(target=write_db, args=([next(_api)]), kwargs={})
        thr.start() # Will run "foo"
        tl.append(thr)
    for i in tl:
        i.join()
    now = time.time() - start
    sum += now
    n += 1
    print "Avg:{} Now:{}".format(sum/n, now)

Read

import pickle
import time
import requests
def load_uuid():
    while True:
        with open("uuid.txt", "rb") as f:
            for i in pickle.load(f):
                yield i
def load_name():
    _uuid = load_uuid()
    services = ['107','108','109','129','134','133','134','191','http_src_based_glid_drop','http_blacklist_drop',
        'http_tcp_service_drop','http_service_drop','http_auth_drop','http_port_glid_drop','http_tcp_auth_drop']
    while True:
        for service in services:
            uuid = next(_uuid)
            yield "sum:{}.{}".format(uuid, service)

url = "http://localhost:6081/api/query"
_name = load_name()
suc, err = 0,0
t, epoch = 0,0
while True:
    start = time.time()
    epoch += 1
    with requests.session() as client:
        for i in xrange(10):
            try:
                data = {
                    'start': '365d-ago', 
                    'noAnnotations': True, 
                    'm': [next(_name)], 
                    'globalAnnotations': False, 
                    'msResolution': False, 
                    'showTSUIDs': False,
                    #'downsample': '10d-avg'
                }
                r = client.get(url, params=data)
                assert r.status_code is 200
                suc += 1
            except KeyboardInterrupt:
                raise Exception("stop")
            except Exception as e:
                print e.message
                err += 1
    t_now = time.time() - start
    t += t_now
    print "Success:{} Error:{} Time:{} Avg:{}".format(suc, err, t_now, t/epoch)