Celery: ์ƒ๋Ÿฌ๋ฆฌ + ์ƒ๋Ÿฌ๋ฆฌ๊ฐ€ ์ž‘๋™ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค

์— ๋งŒ๋“  2017๋…„ 02์›” 22์ผ  ยท  3์ฝ”๋ฉ˜ํŠธ  ยท  ์ถœ์ฒ˜: celery/celery

์ฒดํฌ๋ฆฌ์ŠคํŠธ

  • [ ] ๋ฌธ์ œ์— celery -A proj report ์˜ ์ถœ๋ ฅ์„ ํฌํ•จํ–ˆ์Šต๋‹ˆ๋‹ค.
    ์†Œํ”„ํŠธ์›จ์–ด -> ์…€๋Ÿฌ๋ฆฌ:4.0.2(์ž ์žฌ์  ํ˜ธ์ถœ) kombu:4.0.2 py:3.6.0
    ๋‹น๊ตฌ:3.5.0.2 py-amqp:2.1.4
    ํ”Œ๋žซํผ -> ์‹œ์Šคํ…œ:๋‹ค์œˆ ์•„์น˜:64๋น„ํŠธ ์ž„ํ”„:CPython
    ๋กœ๋” -> celery.loaders.app.AppLoader
    ์„ค์ • -> ์ „์†ก:amqp ๊ฒฐ๊ณผ:django-db).
  • [ ] Celery์˜ master ๋ถ„๊ธฐ์— ๋ฌธ์ œ๊ฐ€ ์žˆ์Œ์„ ํ™•์ธํ–ˆ์Šต๋‹ˆ๋‹ค.

์žฌํ˜„ ๋‹จ๊ณ„

1 .deploy_tomcat2.py
`.AnsibleApi์—์„œ CallApi ๊ฐ€์ ธ์˜ค๊ธฐ

def django_process(jira_num):
์„œ๋ฒ„ = '10.10.51.30'
์ด๋ฆ„ = 'abc'
ํฌํŠธ = 11011
์ฝ”๋“œ = 'efs'
jdk = '1.12.13'
jvm = 'xxxx'

if str.isdigit(jira_num):
    # import pdb
    # pdb.set_trace()
    call = CallApi(server,name,port,code,jdk,jvm)
    return call.run_task()

`

  1. AnsibleApi.py
    `#!/usr/bin/env ํŒŒ์ด์ฌ

- - ์ฝ”๋”ฉ: utf-8 - -

๊ฐ€์ ธ์˜ค๊ธฐ ๋กœ๊น…
.Logger์—์„œ ๊ฐ€์ ธ์˜ค๊ธฐ ๋กœ๊ฑฐ
django.conf ๊ฐ€์ ธ์˜ค๊ธฐ ์„ค์ •์—์„œ
์ปฌ๋ ‰์…˜์—์„œ ๋ช…๋ช…๋œ ํŠœํ”Œ ๊ฐ€์ ธ์˜ค๊ธฐ
fromsible.parsing.dataloader ๊ฐ€์ ธ์˜ค๊ธฐ DataLoader
VariableManager ๊ฐ€์ ธ์˜ค๊ธฐ
fromsible.inventory ๊ฐ€์ ธ์˜ค๊ธฐ ์ธ๋ฒคํ† ๋ฆฌ
fromsible.playbook.play ๊ฐ€์ ธ์˜ค๊ธฐ ์žฌ์ƒ
fromsible.executor.task_queue_manager import TaskQueueManager
fromsible.plugins.callback import CallbackBase

๋กœ๊ทธ = ๋กœ๊ฑฐ('/tmp/auto_deploy_tomcat.log',logging.INFO)

ํด๋ž˜์Šค ๊ฒฐ๊ณผ์ฝœ๋ฐฑ(์ฝœ๋ฐฑ๋ฒ ์ด์Šค):
def __init__(self, args, * kwargs):
super(ResultCallback, self).__init__( args, * kwargs)
self.host_ok = {}
self.host_unreachable = {}
self.host_failed = {}

def v2_runner_on_unreachable(self, result):
    self.host_unreachable[result._host.get_name()] = result

def v2_runner_on_ok(self, result, *args, **kwargs):
    self.host_ok[result._host.get_name()] = result

def v2_runner_on_failed(self, result, *args, **kwargs):
    self.host_failed[result._host.get_name()] = result

ํด๋ž˜์Šค CallApi(๊ฐ์ฒด):
์‚ฌ์šฉ์ž = ์„ค์ •.SSH_USER
ssh_private_key_file = ์„ค์ •.SSH_PRIVATE_KEY_FILE
๊ฒฐ๊ณผ_์ฝœ๋ฐฑ = ๊ฒฐ๊ณผ์ฝœ๋ฐฑ()
์˜ต์…˜ = namedtuple('์˜ต์…˜',
['์—ฐ๊ฒฐ', '๋ชจ๋“ˆ_๊ฒฝ๋กœ', '๊ฐœ์ธํ‚ค_ํŒŒ์ผ', 'ํฌํฌ', '๋˜๊ธฐ', '๋น„์ปด_๋ฉ”์†Œ๋“œ',
'์‚ฌ์šฉ์ž ๋˜๊ธฐ', 'ํ™•์ธ'])

def __init__(self,ip,name,port,code,jdk,jvm):
    self.ip = ip
    self.name = name
    self.port = port
    self.code = code
    self.jdk = jdk
    self.jvm = jvm
    self.results_callback = ResultCallback()
    self.results_raw = {}

def _gen_user_task(self):
    tasks = []
    deploy_script = 'autodeploy/abc.sh'
    dst_script = '/tmp/abc.sh'
    cargs = dict(src=deploy_script, dest=dst_script, owner=self.user, group=self.user, mode='0755')
    args = "%s %s %d %s %s '%s'" % (dst_script, self.name, self.port, self.code, self.jdk, self.jvm)
    tasks.append(dict(action=dict(module='copy', args=cargs),register='shell_out'))
    tasks.append(dict(action=dict(module='debug', args=dict(msg='{{shell_out}}'))))
    # tasks.append(dict(action=dict(module='command', args=args)))
    # tasks.append(dict(action=dict(module='command', args=args), register='result'))
    # tasks.append(dict(action=dict(module='debug', args=dict(msg='{{result.stdout}}'))))
    self.tasks = tasks

def _set_option(self):
    self._gen_user_task()

    self.variable_manager = VariableManager()
    self.loader = DataLoader()
    self.options = self.Options(connection='smart', module_path=None, private_key_file=self.ssh_private_key_file, forks=None,
                                become=True, become_method='sudo', become_user='root', check=False)
    self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=[self.ip])
    self.variable_manager.set_inventory(self.inventory)

    play_source = dict(
    name = "auto deploy tomcat",
        hosts = self.ip,
        remote_user = self.user,
        gather_facts='no',
        tasks = self.tasks
    )
    self.play = Play().load(play_source, variable_manager=self.variable_manager, loader=self.loader)

def run_task(self):
    self.results_raw = {'success':{}, 'failed':{}, 'unreachable':{}}
    tqm = None
    from celery.contrib import rdb
    rdb.set_trace()
    self._set_option()
    try:
        tqm = TaskQueueManager(
            inventory=self.inventory,
            variable_manager=self.variable_manager,
            loader=self.loader,
            options=self.options,
            passwords=None,
            stdout_callback=self.results_callback,
        )
        result = tqm.run(self.play)
    finally:
        if tqm is not None:
            tqm.cleanup()

    for host, result in self.results_callback.host_ok.items():
        self.results_raw['success'][host] = result._result

    for host, result in self.results_callback.host_failed.items():
        self.results_raw['failed'][host] = result._result

    for host, result in self.results_callback.host_unreachable.items():
        self.results_raw['unreachable'][host]= result._result
    Log.info("result is :%s" % self.results_raw)
    return self.results_raw

3.tasks.py from __future__ import absolute_import, unicode_literals
์…€๋Ÿฌ๋ฆฌ ๊ฐ€์ ธ์˜ค๊ธฐ shared_task์—์„œ
.deploy_tomcat2์—์„œ django_process ๊ฐ€์ ธ์˜ค๊ธฐ

ROM celery.contrib ๊ฐ€์ ธ์˜ค๊ธฐ rdb

@shared_task
def ๋ฐฐํฌ(์ˆซ์ž):
#return 'hello world {0}'.format(num)
#rdb.set_trace()

return django_process(num)`

์˜ˆ์ƒ๋˜๋Š” ํ–‰๋™

django console :python manage.py shell ์—์„œ deploy('123') , ์ฝ”๋“œ ํ˜ธ์ถœ ansibleApi๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ abc.sh๋ฅผ dst ์„œ๋ฒ„์— ์„ฑ๊ณต์ ์œผ๋กœ ๋ณต์‚ฌํ•  ์ˆ˜ ์žˆ์Šต๋‹ˆ๋‹ค.
ํ•˜์ง€๋งŒ 'deploy.apply_async(args=['1334'], queue='queue.ops.deploy', routing_key='ops.deploy')'๋ฅผ ์‚ฌ์šฉํ•  ๋•Œ ์ž‘๋™ํ•˜์ง€ ์•Š์•˜์Šต๋‹ˆ๋‹ค.

๋ฉฐ์น  ๊ฒ€์ƒ‰ํ–ˆ๋Š”๋ฐ ์•„์ง๋„ ์•ˆ๋˜๋„ค์š”.

์‹ค์ œ ํ–‰๋™

AnsibleApi๋Š” ๋นˆ ์‚ฌ์ „์„ ๋ฐ˜ํ™˜ํ•ฉ๋‹ˆ๋‹ค:{'์„ฑ๊ณต': {}, '์‹คํŒจ': {}, '์—ฐ๊ฒฐํ•  ์ˆ˜ ์—†์Œ': {}}

๊ฐ€์žฅ ์œ ์šฉํ•œ ๋Œ“๊ธ€

์ด ๋ฌธ์ œ๋ฅผ ํ•ด๊ฒฐํ•˜๋Š” ๋‘ ๊ฐ€์ง€ ๋ฐฉ๋ฒ•์ด ์žˆ์Šต๋‹ˆ๋‹ค. ์ฃผ์žฅ ๋น„ํ™œ์„ฑํ™”:
1.์—ฌ๊ธฐ์„œ ์…€๋Ÿฌ๋ฆฌ๋Š” ๋‚ด๋ณด๋‚ด๊ธฐ PYTHONOPTIMIZE=1๋กœ ์„ค์ •ํ•˜๊ฑฐ๋‚˜ ์ด ๋งค๊ฐœ๋ณ€์ˆ˜๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ ์…€๋Ÿฌ๋ฆฌ๋ฅผ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. -O OPTIMIZATION
2. python ํŒจํ‚ท ๋‹ค์ค‘ ์ฒ˜๋ฆฌ process.py ๋ผ์ธ 102 ๋น„ํ™œ์„ฑํ™”:
assert not _current_process._config.get('daemon'), \ 'daemonic processes are not allowed to have children'

๋ชจ๋“  3 ๋Œ“๊ธ€

์ด ๋ฌธ์ œ๋ฅผ ํ•ด๊ฒฐํ•˜๋Š” ๋‘ ๊ฐ€์ง€ ๋ฐฉ๋ฒ•์ด ์žˆ์Šต๋‹ˆ๋‹ค. ์ฃผ์žฅ ๋น„ํ™œ์„ฑํ™”:
1.์—ฌ๊ธฐ์„œ ์…€๋Ÿฌ๋ฆฌ๋Š” ๋‚ด๋ณด๋‚ด๊ธฐ PYTHONOPTIMIZE=1๋กœ ์„ค์ •ํ•˜๊ฑฐ๋‚˜ ์ด ๋งค๊ฐœ๋ณ€์ˆ˜๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ ์…€๋Ÿฌ๋ฆฌ๋ฅผ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. -O OPTIMIZATION
2. python ํŒจํ‚ท ๋‹ค์ค‘ ์ฒ˜๋ฆฌ process.py ๋ผ์ธ 102 ๋น„ํ™œ์„ฑํ™”:
assert not _current_process._config.get('daemon'), \ 'daemonic processes are not allowed to have children'

์ด ๋ฌธ์ œ๋ฅผ ํ•ด๊ฒฐํ•˜๋Š” ๋‘ ๊ฐ€์ง€ ๋ฐฉ๋ฒ•์ด ์žˆ์Šต๋‹ˆ๋‹ค. ์ฃผ์žฅ ๋น„ํ™œ์„ฑํ™”:
1.์—ฌ๊ธฐ์„œ ์…€๋Ÿฌ๋ฆฌ๋Š” ๋‚ด๋ณด๋‚ด๊ธฐ PYTHONOPTIMIZE=1๋กœ ์„ค์ •ํ•˜๊ฑฐ๋‚˜ ์ด ๋งค๊ฐœ๋ณ€์ˆ˜๋ฅผ ์‚ฌ์šฉํ•˜์—ฌ ์…€๋Ÿฌ๋ฆฌ๋ฅผ ์‹œ์ž‘ํ•ฉ๋‹ˆ๋‹ค. -O OPTIMIZATION
2. python ํŒจํ‚ท ๋‹ค์ค‘ ์ฒ˜๋ฆฌ process.py ๋ผ์ธ 102 ๋น„ํ™œ์„ฑํ™”:
assert not _current_process._config.get('daemon'), \ '๋ฐ๋ชฌ ํ”„๋กœ์„ธ์Šค๋Š” ์ž์‹์„ ๊ฐ€์งˆ ์ˆ˜ ์—†์Šต๋‹ˆ๋‹ค'

@Xuexiang825 ์ฃ„์†กํ•ฉ๋‹ˆ๋‹ค. ์ž‘๋™ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค. ์ €๋Š” Python 3.6.3๊ณผ ansible 2.4.2.0, celery 4.1.0์„ ์‚ฌ์šฉํ•ฉ๋‹ˆ๋‹ค. ๋‚ด ํ”Œ๋ ˆ์ด๋ถ ์‹คํ–‰๊ธฐ๋Š” ๋ฉ”์‹œ์ง€๋ฅผ ํ‘œ์‹œํ•˜์ง€๋งŒ ์•„๋ฌด ์ž‘์—…๋„ ์ˆ˜ํ–‰ํ•˜์ง€ ์•Š์Šต๋‹ˆ๋‹ค.๋˜ํ•œ -O OPTIMIZATION ๋งค๊ฐœ๋ณ€์ˆ˜์— ๋Œ€ํ•ด ์•Œ๋ ค์ฃผ์‹ค ์ˆ˜ ์žˆ์Šต๋‹ˆ๊นŒ?

@Xuexiang825 ๋ฐฉ๊ธˆ ์ด ๊ธฐ์‚ฌ ์—์„œ ๊ท€ํ•˜์˜

์ด ํŽ˜์ด์ง€๊ฐ€ ๋„์›€์ด ๋˜์—ˆ๋‚˜์š”?
0 / 5 - 0 ๋“ฑ๊ธ‰