如何以较少的延迟动态格式化dict嵌套列表

Question

我需要你的专业知识来简化嵌套字典的格式设置。我有输入信号列表，需要根据分钟精度在u_id和时间戳字段上分组，并转换为各自的输出格式。我已经发布了我尝试过的格式。我需要尽可能快地格式化和处理它，因为这涉及到时间复杂性。非常感谢你的帮助

代码片段

final_output = []

sorted_signals = sorted(signals, key=lambda x: (x['u_id'], str(x['start_ts'])[0:8]))

data = itertools.groupby(sorted_signals, key=lambda x: (x['u_id'], calendar.timegm(time.strptime(datetime.utcfromtimestamp(x['start_ts']).strftime('%Y-%m-%d-%H:%M'),'%Y-%m-%d-%H:%M'))))

def format_signals(v):
    result =[]
    for i in v:
        temp_dict = {}
        temp_dict.update({'timestamp_utc': i['start_ts']})
        for data in i['sign']:
            temp_dict.update({data['name'].split('.')[0]: data['val']})
        result.append(temp_dict)
    return result


for k, v in data:

    output_format = {'ui_id': k[0], 'minute_utc': datetime.fromtimestamp(int(k[1])), 'data': format_signals(v),
                'processing_timestamp_utc': datetime.strptime(datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),"%Y-%m-%d %H:%M:%S")}
    final_output.append(output_format)

print(final_output)

输入

signals = [
       {'c_id': '1234', 'u_id': 288, 'f_id': 331,
        'sign': [{'name': 'speed', 'val': 9},
                    {'name': 'pwr', 'val': 1415}], 'start_ts': 1598440244,
        'crt_ts': 1598440349, 'map_crt_ts': 1598440351, 'ca_id': 'AT123', 'c_n': 'demo',
        'msg_cnt': 2, 'window': 'na', 'type': 'na'},
       {'c_id': '1234', 'u_id': 288, 'f_id': 331,
        'sign': [{'name': 'speed', 'val': 10},
                    {'name': 'pwr', 'val': 1416}], 'start_ts': 1598440243,
        'crt_ts': 1598440349, 'map_crt_ts': 1598440351, 'ca_id': 'AT123', 'c_n': 'demo',
        'msg_cnt': 2, 'window': 'na', 'type': 'na'},
       {'c_id': '1234', 'u_id': 287, 'f_id': 331,
        'sign': [{'name': 'speed', 'val': 10},
                    {'name': 'pwr', 'val': 1417}], 'start_ts': 1598440344,
        'crt_ts': 1598440349, 'map_crt_ts': 1598440351, 'ca_id': 'AT123', 'c_n': 'demo',
        'msg_cnt': 2, 'window': 'na', 'type': 'na'},
       {'c_id': '1234', 'u_id': 288, 'f_id': 331,
        'sign': [{'name': 'speed.', 'val': 8.2},
                    {'name': 'pwr', 'val': 925}], 'start_ts': 1598440345,
        'crt_ts': 1598440349, 'map_crt_ts': 1598440351, 'ca_id': 'AT172', 'c_n': 'demo',
        'msg_cnt': 2, 'window': 'na', 'type': 'na'}
       ]

电流输出

   [{
    'ui_id': 287,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 42),
    'data': [{
        'timestamp_utc': 1598440344,
        'speed': 10,
        'pwr': 1417
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}, {
    'ui_id': 288,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 40),
    'data': [{
        'timestamp_utc': 1598440244,
        'speed': 9,
        'pwr': 1415
    }, {
        'timestamp_utc': 1598440243,
        'speed': 10,
        'pwr': 1416
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}, {
    'ui_id': 288,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 42),
    'data': [{
        'timestamp_utc': 1598440345,
        'speed': 8.2,
        'pwr': 925
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}]

所需输出

    [{
    'ui_id': 287,
    'f_id': 311,
    'c_id': 1234,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 42),
    'data': [{
        'timestamp_utc': 1598440344,
        'speed': 10,
        'pwr': 1417
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}, {
    'ui_id': 288,
     'f_id': 311,
    'c_id': 1234,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 40),
    'data': [{
        'timestamp_utc': 1598440244,
        'speed': 9,
        'pwr': 1415
    }, {
        'timestamp_utc': 1598440243,
        'speed': 10,
        'pwr': 1416
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}, {
    'ui_id': 288,
     'f_id': 311,
    'c_id': 1234,
    'minute_utc': datetime.datetime(2020, 8, 26, 16, 42),
    'data': [{
        'timestamp_utc': 1598440345,
        'speed': 8.2,
        'pwr': 925
    }],
    'processing_timestamp_utc': datetime.datetime(2020, 8, 29, 19, 35, 46)
}]

Answer 1

因此，让我们定义一个简单的函数，它将从分组所需的每个对象键中提取：

def extract(obj):
    return obj['u_id'], obj['f_id'], obj['c_id'], obj['start_ts'] // 60 * 60

注意：为了实现“分钟精度”，我将时间戳除以60以缩短秒数，再乘以60以获得有效的时间戳

然后，让我们将对象分组并形成最终列表：

from itertools import groupby
from datetime import datetime
...
final_output = []
for (uid, fid, cid, ts), ss in groupby(sorted(signals, key=extract), extract):
    obj = {
        'ui_id': uid,
        'f_id': fid,
        'c_id': int(cid),
        'minute_utc': datetime.utcfromtimestamp(ts),
        'data': [],
        'processing_timestamp_utc': datetime.utcnow()
    }
    for s in ss:
        obj['data'].append({
            'timestamp_utc': s['start_ts'],
            **{i['name']: i['val'] for i in s['sign']}
        })
    final_output.append(obj)

要以可读形式打印final_output，我们可以使用pprint：

from pprint import pprint
...
pprint(final_output, sort_dicts=False)

Answer 2

也许这可以帮助您以更直接的方式编写代码。如果你只需要遍历信号并将它们组织在一个循环中，也许你不需要更重的排序和分组

当您希望根据u_id收集信号时，字典可以方便地为每个u_id获取一个条目。这可以做到这一点，您只需添加创建基于此有组织的信号记录的输出：

organized = {}
      
for s in signals:
  u_id = s['u_id']
  
  entry = organized.get(u_id, None)
  if entry is None:
    entry = []
    organized[u_id] = entry  
  entry.append(s)      

pprint.pprint(organized)

在那里是可执行的，输出粘贴在下面，https://repl.it/repls/ShallowQuintessentialInteger

{287: [{'c_id': '1234',
        'c_n': 'demo',
        'ca_id': 'AT123',
        'crt_ts': 1598440349,
        'f_id': 331,
        'map_crt_ts': 1598440351,
        'msg_cnt': 2,
        'sign': [{'name': 'speed', 'val': 10}, {'name': 'pwr', 'val': 1417}],
        'start_ts': 1598440344,
        'type': 'na',
        'u_id': 287,
        'window': 'na'}],
 288: [{'c_id': '1234',
        'c_n': 'demo',
        'ca_id': 'AT123',
        'crt_ts': 1598440349,
        'f_id': 331,
        'map_crt_ts': 1598440351,
        'msg_cnt': 2,
        'sign': [{'name': 'speed', 'val': 9}, {'name': 'pwr', 'val': 1415}],
        'start_ts': 1598440244,
        'type': 'na',
        'u_id': 288,
        'window': 'na'},
       {'c_id': '1234',
        'c_n': 'demo',
        'ca_id': 'AT123',
        'crt_ts': 1598440349,
        'f_id': 331,
        'map_crt_ts': 1598440351,
        'msg_cnt': 2,
        'sign': [{'name': 'speed', 'val': 10}, {'name': 'pwr', 'val': 1416}],
        'start_ts': 1598440243,
        'type': 'na',
        'u_id': 288,
        'window': 'na'},
       {'c_id': '1234',
        'c_n': 'demo',
        'ca_id': 'AT172',
        'crt_ts': 1598440349,
        'f_id': 331,
        'map_crt_ts': 1598440351,
        'msg_cnt': 2,
        'sign': [{'name': 'speed.', 'val': 8.2}, {'name': 'pwr', 'val': 925}],
        'start_ts': 1598440345,
        'type': 'na',
        'u_id': 288,
        'window': 'na'}]}