import multiprocessing as mp
from copy import deepcopy
# import ray
from frozendict import frozendict
from joblib import parallel_backend, delayed, Parallel
from dsmac.runhistory.runhistory import RunHistory
from dsmac.tae.execute_ta_run import ExecuteTARun
class Distributer():
def __init__(self, tae_runner: ExecuteTARun = None, n_jobs=1):
self.n_jobs = self.parse_n_jobs(n_jobs)
self.tae_runner = tae_runner
def run(self, to_run, runhistory: RunHistory):
n_jobs = self.n_jobs
ans = []
for i in range(0, len(to_run), n_jobs):
ret = self._run(to_run[i:i + n_jobs])
ans.extend(ret)
self.add_runhistory(to_run, ans, runhistory)
return ans
def _run(self, to_run: list) -> list:
raise NotImplementedError()
def add_runhistory(self, to_run, ans, runhistory):
for config, (status, cost, dur, res) in zip(to_run, ans):
if not runhistory.get_runs_for_config(config):
runhistory.add(
config,
cost,
dur,
status
)
return runhistory
def parse_n_jobs(self, n_jobs):
if n_jobs == 0:
return 1
elif n_jobs < 0:
return mp.cpu_count() + 1 + n_jobs
else:
return n_jobs
# @ray.remote
# def ray_run_fun(tae_runner, challenger):
# status, cost, dur, res = tae_runner.start(
# config=challenger,
# instance=None)
# return (status, cost, dur, res)
#
#
# class RayDistributer(Distributer):
# def __init__(self, tae_runner: ExecuteTARun = None,
# ray_config: dict = frozendict(), n_jobs=-1):
# super(RayDistributer, self).__init__(tae_runner, n_jobs)
#
# ray.init(**ray_config)
# print('init ray')
# def __del__(self):
# print('del ray')
# ray.shutdown()
# def _run(self, to_run):
# ans = [ray_run_fun.remote(self.tae_runner, item) for item in to_run]
# ray_ans = ray.get(ans) # (status, cost, dur, res)
# return ray_ans
def mp_run_fun(tae_runner, config, inst): # ,q
ans = tae_runner.start(config, inst)
return ans
# q.put((config,ans))
class MultiProcessDistributer(Distributer):
def _run(self, to_run):
processes = []
q = mp.Queue()
for config in to_run:
p = mp.Process(target=mp_run_fun, args=(self.tae_runner, config, None, q))
processes.append(p)
p.start()
for p in processes:
p.join()
results = [q.get() for j in processes]
anss = [result[1] for result in results]
return anss
global g_tae_runner
def joblib_run_fun(challenger):
status, cost, dur, res = g_tae_runner.start(
config=challenger,
instance=None)
return (status, cost, dur, res)
class JoblibDistributer(Distributer):
def __init__(self, *args, **kwargs):
super(JoblibDistributer, self).__init__(*args, **kwargs)
global g_tae_runner
g_tae_runner = self.tae_runner
def _run(self, to_run: list) -> list:
with parallel_backend(backend="multiprocessing", n_jobs=-1):
# ans=Parallel()(delayed(self.tae_runner.start)(x,None) for x in to_run)
ans = Parallel()(delayed(mp_run_fun)(deepcopy(self.tae_runner), x, None) for x in to_run)
return ans
class SingleDistributer(Distributer):
def __init__(self, *args, **kwargs):
super(SingleDistributer, self).__init__(*args, **kwargs)
self.n_jobs = 1
def _run(self, to_run):
ans = [self.tae_runner.start(item, None) for item in to_run]
return ans
dsmac中废弃的mapreduce方案
原创wx61090d1892228 博主文章分类:automl ©著作权
©著作权归作者所有:来自51CTO博客作者wx61090d1892228的原创作品,请联系作者获取转载授权,否则将追究法律责任
提问和评论都可以,用心的回复会被更多人看到
评论
发布评论
相关文章
-
@OnLifecycleEnvent 被废弃,替代方案更简单
近期 androidx.lifecycle 发布了 2.4.0 版本,此次更新中 @OnLifecycleEvent 注解被废弃,官方建议使用 DefaultLifecycleOvserver
Android Android Jetpack ide android java