CMDB_Agent_ssh版本分析
Jeff的技术栈 人气:1
[TOC]
# CMDB_Agent+ssh版本+server端
# CMDB_Agent版本
## CMDB概念
```
CMDB: Configure Manage DataBase
中文：配置管理数据库。
主要的作用是：收集服务器的基础信息(包括：服务器的主机名，ip，操作系统版本，磁盘，CPU等信息)，将来提供给子系统(代码发布，工单系统等)数据
```
## CMDB_Agent介绍
```
其本质上就是在各个服务器上执行subprocess.getoutput()命令，然后将每台机器上执行的结果，返回给主机API，然后主机API收到这些数据之后，放入到数据库中，最终通过web界面展现给用户
优点：速度快
缺点：需要为每台服务器步数一个Agent的程序
```
### agent方案
```
将待采集的服务器看成一个agent，然后再服务器上使用python的subprocess模块执行linux相关的命令，然后分析得到的结果，将分析得到的结果通过requests模块发送给API，API获取到数据之后，进行二次比对数据，最后将比对的结果存入到数据库中，最后django起一个webserver从数据库中将数据获取出来，供用户查看
```
### ssh类方案
```
在中控机服务器上安装一个模块叫paramiko模块，通过这个模块登录到带采集的服务器上，然后执行相关的linux命令，最后返回执行的结果，将分析得到的结果通过requests模块发送给API，API获取到数据之后，进行二次比对数据，最后将比对的结果存入到数据库中，最后django起一个webserver从数据库中将数据获取出来，供用户查看
```
### 相比较
```
agent方案
优点:不需要额外的增加中控机。
缺点：每新增一台服务器，就需要额外部署agent脚本。使用场景是：服务器多的情况 (1000台以上)
ssh方案
优点：不需要额外的部署脚本。
缺点：速度比较慢。使用场景是：服务器少 (1000台往下)
```
# client端
## 架构目录
## bin-start.py 启动文件
```python
from src.srcipt import run
if __name__ == '__main__':
run()
```
## conf-config.py 自定义配置文件
模仿Django的setting,常用的配置写在这里面。不常用的写在global_settings.py中。
加载顺寻：先加载全局的。再加载局部的
```python
USER = 'root'
MODE = 'agent'
DEBUG = True # True:代表是开发测试阶段 False:代表是上现阶段
import os
BASEDIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PLUGINS_DICT = {
'basic': 'src.plugins.basic.Basic',
'cpu': 'src.plugins.cpu.Cpu',
'disk': 'src.plugins.disk.Disk',
# 'memory': 'src.plugins.memory.Memory',
}
APIURL = 'http://127.0.0.1:8000/api/'
```
## files 开发测试的文件
DEBUT=True时为测试阶段，用files的测试数据
## lib-config-global_settings.py 全局配置的文件
```
pass
```
## lib-config-conf.py 读取配置的文件
全局配置放在前面先加载，自定义配置的放在后面后加载。自定义配置了就用自定义的(覆盖)，没有配置久用全局的
```python
from conf import config
from . import global_settings
class mySettings():
def __init__(self):
# print('aa:', dir(global_settings))
# print('bb:', dir(config))
# 全局配置
for k in dir(global_settings):
if k.isupper():
v = getattr(global_settings, k)
setattr(self, k, v)
# 自定义配置
for k in dir(config):
if k.isupper():
v = getattr(config, k)
setattr(self, k, v)
settings = mySettings()
```
## src-plugins-__init__.py 核心文件
```python
from lib.config.conf import settings
import importlib
class PluginsManager():
def __init__(self, hostname=None):
self.plugins_dict = settings.PLUGINS_DICT
self.debug = settings.DEBUG
self.hostname = hostname
# 1.采集数据
def execute(self):
response = {}
for k, v in self.plugins_dict.items():
'''
k: basic
v: src.plugins.basic.Basic
'''
res = {'status':None, 'data':None}
try:
# 2.循环导入(字符串路径)
moudle_path, class_name = v.rsplit('.', 1) # ['src.plugins.basic','Basic']
# 用importlib.import_module()导入字符串路径
m = importlib.import_module(moudle_path)
# 3.导入类
cls = getattr(m, class_name)
# 循环执行鸭子类型的process方法,command_func函数的内存地址传过去，把debug传过去
ret = cls().process(self.command_func, self.debug)
res['status'] = 10000
res['data'] = ret
response[k] = res
except Exception as e:
import traceback
res['status'] = 10001
res['data'] = '错误信息：%s'%(traceback.format_exc())
response[k] = res
return response
# 真正的连接，执行命令，返回结果的函数。命令变成参数
def command_func(self, cmd):
if settings.MODE == 'agent':
import subprocess
res = subprocess.getoutput(cmd)
return res
else:
import paramiko
# 创建SSH对象
ssh = paramiko.SSHClient()
# 允许连接不再know_hosts文件中的主机
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# 连接服务器
ssh.connect(hostname=self.hostname, port=22, username='root', password='123456')
# 执行命令
stdin, stdout, stderr = ssh.exec_command(cmd)
# 获取命令结果
result = stdout.read()
# 关闭连接
ssh.close()
return result
```
## src-plugins-basic.py 查看硬件信息
```python
from conf import config
class Basic(object):
def process(self, command_func, debug):
if debug:
output = {
'os_platform': "linux",
'os_version': "CentOS release 6.6 (Final)\nKernel \r on an \m",
'hostname': 'c1.com'
}
else:
output = {
'os_platform': command_func("uname").strip(),
'os_version': command_func("cat /etc/issue").strip().split('\n')[0],
'hostname': command_func("hostname").strip(),
}
return output
```
## src-plugins-cpu.py 查看cpu属性
```python
import os
from lib.config.conf import settings
class Cpu():
def __init__(self):
pass
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'files/cpuinfo.out'), 'r', encoding='utf-8').read()
else:
output = command_func("cat /proc/cpuinfo")
return self.parse(output)
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {'cpu_count': 0, 'cpu_physical_count': 0, 'cpu_model': ''}
cpu_physical_set = set()
content = content.strip()
for item in content.split('\n\n'):
for row_line in item.split('\n'):
key, value = row_line.split(':')
key = key.strip()
if key == 'processor':
response['cpu_count'] += 1
elif key == 'physical id':
cpu_physical_set.add(value)
elif key == 'model name':
if not response['cpu_model']:
response['cpu_model'] = value
response['cpu_physical_count'] = len(cpu_physical_set)
return response
```
## src-plugins-disk.py 查看磁盘信息
```python
# 采集磁盘信息
from lib.config.conf import settings
import os
import re
class Disk(object):
def __init__(self):
pass
def process(self, command_func, debug):
if debug:
output = open(os.path.join(settings.BASEDIR, 'fileshttps://img.qb5200.com/download-x/disk.out'), 'r', encoding='utf-8').read()
else:
output = command_func('MegaCli -PDList -aALL') # radi 卡 磁盘阵列
return self.parse(output) # 调用过滤的函数
# 过滤函数，对字符串的处理过滤
def parse(self, content):
"""
解析shell命令返回结果
:param content: shell 命令结果
:return:解析后的结果
"""
response = {}
result = []
for row_line in content.split("\n\n\n\n"):
result.append(row_line)
for item in result:
temp_dict = {}
for row in item.split('\n'):
if not row.strip():
continue
if len(row.split(':')) != 2:
continue
key, value = row.split(':')
name = self.mega_patter_match(key)
if name:
if key == 'Raw Size':
raw_size = re.search('(\d+\.\d+)', value.strip())
if raw_size:
temp_dict[name] = raw_size.group()
else:
raw_size = '0'
else:
temp_dict[name] = value.strip()
if temp_dict:
response[temp_dict['slot']] = temp_dict
return response
@staticmethod
def mega_patter_match(needle):
grep_pattern = {'Slot': 'slot', 'Raw Size': 'capacity', 'Inquiry': 'model', 'PD Type': 'pd_type'}
for key, value in grep_pattern.items():
if needle.startswith(key):
return value
return False
```
# server端
## 架构目录
```
服务端目录结构的设计 django的app
- api ： 负责接收数据， 并且对比入库的
- backend: 前端数据的展示
- repository: 负责数据表的设计
```
## 配置
```
数据库，App注册等省略
```
## repository-models.py 表设计
```python
from django.db import models
# Create your models here.
class UserProfile(models.Model):
"""
用户信息
"""
name = models.CharField(u'姓名', max_length=32)
email = models.EmailField(u'邮箱')
phone = models.CharField(u'座机', max_length=32)
mobile = models.CharField(u'手机', max_length=32)
class Meta:
verbose_name_plural = "用户表"
def __str__(self):
return self.name
class UserGroup(models.Model):
"""
用户组
"""
name = models.CharField(max_length=32, unique=True)
users = models.ManyToManyField('UserProfile')
class Meta:
verbose_name_plural = "用户组表"
def __str__(self):
return self.name
class BusinessUnit(models.Model):
"""
业务线
"""
name = models.CharField('业务线', max_length=64, unique=True)
contact = models.ForeignKey('UserGroup', verbose_name='业务联系人', related_name='c', on_delete=models.CASCADE)
class Meta:
verbose_name_plural = "业务线表"
def __str__(self):
return self.name
class IDC(models.Model):
"""
机房信息
"""
name = models.CharField('机房', max_length=32)
floor = models.IntegerField('楼层', default=1)
class Meta:
verbose_name_plural = "机房表"
def __str__(self):
return self.name
class Server(models.Model):
"""
服务器信息
"""
device_type_choices = (
(1, '服务器'),
(2, '交换机'),
(3, '防火墙'),
)
device_status_choices = (
(1, '上架'),
(2, '在线'),
(3, '离线'),
(4, '下架'),
)
device_type_id = models.IntegerField('服务器类型', choices=device_type_choices, default=1)
device_status_id = models.IntegerField('服务器状态', choices=device_status_choices, default=1)
cabinet_num = models.CharField('机柜号', max_length=30, null=True, blank=True)
cabinet_order = models.CharField('机柜中序号', max_length=30, null=True, blank=True)
idc = models.ForeignKey('IDC', verbose_name='IDC机房', null=True, blank=True, on_delete=models.CASCADE)
business_unit = models.ForeignKey('BusinessUnit', verbose_name='属于的业务线', null=True, blank=True,
on_delete=models.CASCADE)
hostname = models.CharField('主机名', max_length=128, unique=True)
sn = models.CharField('SN号', max_length=64, db_index=True, blank=True)
manufacturer = models.CharField(verbose_name='制造商', max_length=64, null=True, blank=True)
model = models.CharField('型号', max_length=64, null=True, blank=True)
manage_ip = models.GenericIPAddressField('管理IP', null=True, blank=True)
os_platform = models.CharField('系统', max_length=16, null=True, blank=True)
os_version = models.CharField('系统版本', max_length=16, null=True, blank=True)
cpu_count = models.IntegerField('CPU个数', null=True, blank=True)
cpu_physical_count = models.IntegerField('CPU物理个数', null=True, blank=True)
cpu_model = models.CharField('CPU型号', max_length=128, null=True, blank=True)
create_at = models.DateTimeField(auto_now_add=True, blank=True)
class Meta:
verbose_name_plural = "服务器表"
def __str__(self):
return self.hostname
class Disk(models.Model):
"""
硬盘信息
"""
slot = models.CharField('插槽位', max_length=8)
model = models.CharField('磁盘型号', max_length=32)
capacity = models.CharField('磁盘容量GB', max_length=32)
pd_type = models.CharField('磁盘类型', max_length=32)
server_obj = models.ForeignKey('Server', related_name='disk', on_delete=models.CASCADE)
class Meta:
verbose_name_plural = "硬盘表"
def __str__(self):
return self.slot
class NIC(models.Model):
"""
网卡信息
"""
name = models.CharField('网卡名称', max_length=128)
hwaddr = models.CharField('网卡mac地址', max_length=64)
netmask = models.CharField(max_length=64)
ipaddrs = models.CharField('ip地址', max_length=256)
up = models.BooleanField(default=False)
server_obj = models.ForeignKey('Server', related_name='nic', on_delete=models.CASCADE)
class Meta:
verbose_name_plural = "网卡表"
def __str__(self):
return self.name
class Memory(models.Model):
"""
内存信息
"""
slot = models.CharField('插槽位', max_length=32)
manufacturer = models.CharField('制造商', max_length=32, null=True, blank=True)
model = models.CharField('型号', max_length=64)
capacity = models.FloatField('容量', null=True, blank=True)
sn = models.CharField('内存SN号', max_length=64, null=True, blank=True)
speed = models.CharField('速度', max_length=16, null=True, blank=True)
server_obj = models.ForeignKey('Server', related_name='memory', on_delete=models.CASCADE)
class Meta:
verbose_name_plural = "内存表"
def __str__(self):
return self.slot
class ErrorLog(models.Model):
"""
错误日志,如：agent采集数据错误 或 运行错误
"""
server_obj = models.ForeignKey('Server', null=True, blank=True, on_delete=models.CASCADE)
title = models.CharField(max_length=16)
content = models.TextField()
create_at = models.DateTimeField(auto_now_add=True)
class Meta:
verbose_name_plural = "错误日志表"
def __str__(self):
return self.title
```
## Api-views.py 数据处理
这里只对磁盘信息做了处理
```
数据处理：
'''
老的槽位信息： [1,2,6]
新的槽位信息：[1,2,3,4,5]
处理：增加3，4，5槽位
删除6槽位
检测1，2槽位有无更新磁盘信息
下面的所有事情：分析磁盘的信息与老信息
1.增加了那些槽位，
2.删除了那些槽位
3.更新了那些槽位，记录变更的日志
'''
```
```python
from django.shortcuts import render,HttpResponse
import json
from repository import models
def asset(request):
if request.method == 'POST':
info = json.loads(request.body)
hostname = info['basic']['data']['hostname'] # c1.com
# 每一个服务器的对象，这里固定为c1.com服务器
server_obj = models.Server.objects.filter(hostname=hostname).first()
if not server_obj:
return HttpResponse('服务器为录入')
# 磁盘数据状态码为例
status = info['disk']['status'] # 状态码
if status != 10000:
# 添加错误信息
models.ErrorLog.objects.create(title='错误信息', content=info['disk']['data'], server_obj=server_obj)
return HttpResponse('采集出错！')
'''
老的槽位信息： [1,2,6]
新的槽位信息：[1,2,3,4,5]
处理：增加3，4，5槽位
删除6槽位
检测1，2槽位有无更新磁盘信息
下面的所有事情：分析磁盘的信息与老信息
1.增加了那些槽位，
2.删除了那些槽位
3.更新了那些槽位，记录变更的日志
'''
new_disk_info = info['disk']['data'] # 新的磁盘信息
print(new_disk_info)
old_disk_info = models.Disk.objects.filter(server_obj=server_obj).all() # 老的磁盘信息
# 集合去重
new_slot = set(new_disk_info.keys())
old_slot = []
for obj in old_disk_info:
old_slot.append(obj.slot)
old_slot = set(old_slot)
print(new_slot)
print(old_slot)
# 增加的槽位数据
add_slot = new_slot.difference(old_slot)
if add_slot:
for slot in add_slot:
### {'slot': '3', 'pd_type': 'SATA', 'capacity': '476.939', 'model': 'S1AXNSAF912433K Samsung SSD 840 PRO Series DXM06B0Q'},
add_disk_info = new_disk_info[slot]
add_disk_info['server_obj'] = server_obj
#### 可以将增加的变更 {2,3,4,5} 数据记录到变更日志表中
models.Disk.objects.create(**add_disk_info)
# 删除的槽位数据
del_slot = old_slot.difference(new_slot)
if del_slot:
models.Disk.objects.filter(server_obj=server_obj, slot__in=del_slot).delete()
### 将删除的槽位数据记录到变更日志表中
# 更新的槽位数据
up_slot = new_slot.intersection(old_slot)
if up_slot:
for slot in up_slot:
## {'slot': '0', 'pd_type': 'SATA', 'capacity': '279.396', 'model': 'SEAGATE ST300MM0006 LS08S0K2B5NV'}
new_disk_row = new_disk_info[slot]
## obj(slot:0, pd_type:sas, capacity:234,...)
old_disk_row = models.Disk.objects.filter(slot=slot, server_obj=server_obj).first()
for k, new_v in new_disk_row.items():
'''
k: slot, pd_type, capacity, model...
new_v: 0, SATA, 279 , ...
'''
# 利用反射获取
old_v = getattr(old_disk_row, k)
if new_v != old_v:
# 记录变更日志，利用反射添加
setattr(old_disk_row, k, new_v)
old_disk_row.save()
print(info)
return HttpResponse('ok')
else:
print('get')
print(request.body)
return ['c1.com','a2']
```
加载全部内容