需求

機(jī)械硬盤(pán)太慢，但有大的內(nèi)存128G，將數(shù)據(jù)集緩存到內(nèi)存里，用flask作server, requests作client

server

#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright ? 2020 2020-07-15 carryhjr
#
# Distributed under terms of the MIT license.

import base64
from flask import request
from flask import Flask
import os
 
app=Flask(__name__)


def multiprocess_run(process, obj_list, notebook=False):
    import sys
    if notebook is True:
        from tqdm import tqdm_notebook as tqdm
    else:
        from tqdm import tqdm
    from multiprocessing import Pool
    import multiprocessing
    import time
    cpus = multiprocessing.cpu_count()
    pool = Pool(processes=cpus)
    result_list = []
    tic = time.time()
    for result in tqdm(pool.imap(process, obj_list)):
        result_list.append(result)
        sys.stdout.flush()
    toc = time.time()
    print('time waste', toc - tic)
    return result_list
def process(img_path):
    with open(img_path, 'rb') as f:
        string = f.read()
    return (img_path, string)

def save_to_ram(root):
    import glob
    image_paths = glob.glob(root + '/*')[:]
    result_list = multiprocess_run(process, image_paths)
    v = {}
    for result in result_list:
        img_path, byte = result
        v[img_path] = byte
    return v

                            
root = '/home/yons/data/kaggle/global-wheat-detection/global-wheat-detection-coco-format/images'
v = save_to_ram(root)

 
@app.route("/data", methods=['GET'])
def get_frame():
    img_path = request.args.get("img_path")
    #  print(img_path)
    if img_path:
        return v[img_path]

 
if __name__ == "__main__":
    app.run()

client

import glob
img_paths = glob.glob(root + '/*')
import random
img_path = random.choice(img_paths)

import requests
url = "http://127.0.0.1:5000/data"
res = requests.get(url=url, params={'img_path': img_path})
plt.figure()
img = cv2.imdecode(np.frombuffer(res.content, np.uint8), 1)
plt.imshow(img[:,:,::-1])

plt.figure()
img = cv2.imread(img_path)
plt.imshow(img[:,:,::-1])