E-MapReduce gateway host文件记录太多_问答-阿里云开发者社区

问题原因：主集群扩容时，会往gateway下发新增host的任务，但缩容时不会。这个问题已经修复，但存量节点可能也会有提出问题的现象。

操作方案：可在gateway机器上，用ecm-agent python3 跑如下脚本：

from ecm_agent.hosts.hosts import Hosts, HostsEntry

import shutil

import os

import random

import commands

import re

HostEntryPattern1 = 'emr-worker-(\d+)$'

HostEntryPattern2 = 'emr-worker-(\d+).cluster-\d+$'

class HostsFileUtil:

TEMP_HOST_FILE = "/var/lib/ecm-agent/data/host_tmp_"

def __init__(self):

self.temporary_hosts = None

def cleanup_host_file(self):

try:

host_suffix = self._ensure_temporary_hosts()

print("records before: " + str(len(self.temporary_hosts.entries)))

new_entrys = []

for entry in self.temporary_hosts.entries:

if entry.names:

entry.names = self._cleanup_host_names(entry.names)

new_entry = HostsEntry(entry_type=entry.entry_type, address=entry.address, names=entry.names, comment=entry.comment)

new_entrys.append(new_entry)

print("records before: " + str(len(new_entrys)))

self.temporary_hosts.entries = new_entrys

self.temporary_hosts.write()

self._dump_temporary_hosts(host_suffix)

except Exception, e:

print("cleanup_host_entry failed, " + str(e))

def _ensure_temporary_hosts(self):

host_suffix = str(random.randint(0, 1000))

if self.temporary_hosts is None:

if not os.path.exists(self.TEMP_HOST_FILE + host_suffix):

with open(self.TEMP_HOST_FILE + host_suffix, "w+") as temp_hosts:

pass

shutil.copyfile("/etc/hosts", self.TEMP_HOST_FILE + host_suffix)

self.temporary_hosts = Hosts(path=self.TEMP_HOST_FILE + host_suffix)

if not self.temporary_hosts:

raise Exception('init hosts object failed')

return host_suffix

def _dump_temporary_hosts(self, host_suffix):

code, result = commands.getstatusoutput("mv " + self.TEMP_HOST_FILE + host_suffix + " /etc/hosts")

if code != 0:

print(result)

exit(1)

def _cleanup_host_names(self, names):

result = []

temp_names_short = []

temp_names_long = []

for name in names:

matchObj = re.match(HostEntryPattern1, name)

if matchObj:

temp_names_short.append([int(matchObj.group(1)), matchObj.group()])

continue

matchObj = re.match(HostEntryPattern2, name)

if matchObj:

temp_names_long.append([int(matchObj.group(1)), matchObj.group()])

continue

result.append(name)

if temp_names_short:

temp_names_short = sorted(temp_names_short, key=lambda x:x[0])

result.insert(0, temp_names_short[-1][1])

if temp_names_long:

temp_names_long = sorted(temp_names_long, key=lambda x:x[0])

result.insert(0, temp_names_long[-1][1])

return result

if __name__ == "__main__":

hostsUtil = HostsFileUtil()

hostsUtil.cleanup_host_file()

E-MapReduce gateway host文件记录太多

相关课程

相关电子书