python从lis构造嵌套dict

Question

我正在用python将一些糟糕的列表从api和bd输出解析成一个嵌套的json结构，该结构具有一种特定的格式，可供大量前端服务使用。你知道吗

下面的列表是一个示例，每个项目都是一个文件的完整路径。我无法修改此输入，因为它来自遍历数据库的外部服务。目录项不出现在这个列表中，只有文件，文件所在的目录路径是明显的形式，即在下面没有MIPK/DORAS文件存在。示例如下：

"/generic_root/site_store/MIPK/CM.toroidal",
"/generic_root/site_store/MIPK/CM.Supervoid",
"/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest",
"/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign",
"/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes",
"/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant",
"/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral",
"/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking"

以前使用过一个非常慢的函数，但是我现在正在使用下面的代码解析列表。输出格式不是我想要的格式。我在根上附加一个节点时被卡住了。你知道吗

在下面的示例中，它获取一个路径，查找嵌套目录并删除每个文件中存在的根路径，然后创建一个具有适当嵌套结构的节点对象。你知道吗

在将它添加到prev\u节点之后，它将以目录名作为键附加到字典中。你知道吗

import logging
logger = logging.getLogger(__name__)


def main():
    # Initialise
    root_path    = '/generic_root'
    store        = '/site_store'
    file_list    = [
        "/generic_root/site_store/MIPK/CM.toroidal",
        "/generic_root/site_store/MIPK/CM.Supervoid",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking"
    ]
    # Call loop and display results
    viewstore_tree_map = create_dir_tree(file_list, root_path, store)
    logging.info('\n\tView store keys: %s\n\tKeys: %s\n\tDict of store: %s',
                 len(viewstore_tree_map.keys()), viewstore_tree_map.keys(),
                 viewstore_tree_map)


def joiner(delimiter, *args):
    '''
    Joins path strings correctly, unpack before passing args
    '''
    return delimiter.join(list(args))


def create_dir_tree(file_list, root_path, store):
    '''
    File_list [LIST][STR]
    root_path [STR]
    store [STR]
    Return [DICT]
    '''
    node_map = {}
    full_root = root_path+store
    for sub_path in file_list:
        parents = sub_path.replace(full_root, '').split('/')[1:-1]
        prev_node = None
        node = None
        node_path = full_root
        # create tree structure for directory nodes
        for parent in parents:
            node_path = joiner('/', node_path, parent)
            node_exists = 1
            if node_path not in node_map:
                node_exists = 0
                node_map[node_path] = {
                    'name': parent,
                    'data': [],
                    'type': 'dir',
                    'path': node_path,
                }
            node = node_map[node_path]
            # Handles appending previous dict to data field of new dict
            if prev_node and not node_exists:
                prev_node['data'].append(node)
            prev_node = node
            # logger.info(pprint.pprint(prev_node))

        if node:
            node['data'].append({
                'name': sub_path.rsplit('/')[-1],
                'type': 'file',
                'path': sub_path
            })
    return node_map

下面是上述代码的输出。这是一个巨大的内存问题，因为这些列表最终会越来越大。你知道吗

node_map = {
    '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE': {
        'type': 'dir',
        'data': [{
            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant',
            'type': 'file',
            'name': 'CM.tangeant'
        }, {
            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral',
            'type': 'file',
            'name': 'CM.astral'
        }, {
            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking',
            'type': 'file',
            'name': 'CM.forking'
        }],
        'name': 'CRUDE',
        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE'
    },
    '/generic_root/site_store/MIPK/DORAS/CRUDE': {
        'type': 'dir',
        'data': [{
            'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest',
            'type': 'file',
            'name': 'CM.forest'
        }, {
            'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign',
            'type': 'file',
            'name': 'CM.benign'
        }, {
            'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes',
            'type': 'file',
            'name': 'CM.dunes'
        }],
        'name': 'CRUDE',
        'path': '/generic_root/site_store/MIPK/DORAS/CRUDE'
    },
    '/generic_root/site_store/MIPK/DORAS/COMMODITIES': {
        'type': 'dir',
        'data': [{
            'type': 'dir',
            'data': [{
                'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant',
                'type': 'file',
                'name': 'CM.tangeant'
            }, {
                'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral',
                'type': 'file',
                'name': 'CM.astral'
            }, {
                'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking',
                'type': 'file',
                'name': 'CM.forking'
            }],
            'name': 'CRUDE',
            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE'
        }],
        'name': 'COMMODITIES',
        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES'
    },
    '/generic_root/site_store/MIPK': {
        'type': 'dir',
        'data': [{
            'path': '/generic_root/site_store/MIPK/CM.toroidal',
            'type': 'file',
            'name': 'CM.toroidal'
        }, {
            'path': '/generic_root/site_store/MIPK/CM.Supervoid',
            'type': 'file',
            'name': 'CM.Supervoid'
        }, {
            'type': 'dir',
            'data': [{
                'type': 'dir',
                'data': [{
                    'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest',
                    'type': 'file',
                    'name': 'CM.forest'
                }, {
                    'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign',
                    'type': 'file',
                    'name': 'CM.benign'
                }, {
                    'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes',
                    'type': 'file',
                    'name': 'CM.dunes'
                }],
                'name': 'CRUDE',
                'path': '/generic_root/site_store/MIPK/DORAS/CRUDE'
            }, {
                'type': 'dir',
                'data': [{
                    'type': 'dir',
                    'data': [{
                        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant',
                        'type': 'file',
                        'name': 'CM.tangeant'
                    }, {
                        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral',
                        'type': 'file',
                        'name': 'CM.astral'
                    }, {
                        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking',
                        'type': 'file',
                        'name': 'CM.forking'
                    }],
                    'name': 'CRUDE',
                    'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE'
                }],
                'name': 'COMMODITIES',
                'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES'
            }],
            'name': 'DORAS',
            'path': '/generic_root/site_store/MIPK/DORAS'
        }],
        'name': 'MIPK',
        'path': '/generic_root/site_store/MIPK'
    },
    '/generic_root/site_store': {
        'type': 'dir',
        'data': [{
            'type': 'dir',
            'data': [{
                'path': '/generic_root/site_store/MIPK/CM.toroidal',
                'type': 'file',
                'name': 'CM.toroidal'
            }, {
                'path': '/generic_root/site_store/MIPK/CM.Supervoid',
                'type': 'file',
                'name': 'CM.Supervoid'
            }, {
                'type': 'dir',
                'data': [{
                    'type': 'dir',
                    'data': [{
                        'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest',
                        'type': 'file',
                        'name': 'CM.forest'
                    }, {
                        'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign',
                        'type': 'file',
                        'name': 'CM.benign'
                    }, {
                        'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes',
                        'type': 'file',
                        'name': 'CM.dunes'
                    }],
                    'name': 'CRUDE',
                    'path': '/generic_root/site_store/MIPK/DORAS/CRUDE'
                }, {
                    'type': 'dir',
                    'data': [{
                        'type': 'dir',
                        'data': [{
                            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant',
                            'type': 'file',
                            'name': 'CM.tangeant'
                        }, {
                            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral',
                            'type': 'file',
                            'name': 'CM.astral'
                        }, {
                            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking',
                            'type': 'file',
                            'name': 'CM.forking'
                        }],
                        'name': 'CRUDE',
                        'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE'
                    }],
                    'name': 'COMMODITIES',
                    'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES'
                }],
                'name': 'DORAS',
                'path': '/generic_root/site_store/MIPK/DORAS'
            }],
            'name': 'MIPK',
            'path': '/generic_root/site_store/MIPK'
        }],
        'name': 'site_store',
        'path': '/generic_root/site_store'
    },
    '/generic_root/site_store/MIPK/DORAS': {
        'type': 'dir',
        'data': [{
            'type': 'dir',
            'data': [{
                'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest',
                'type': 'file',
                'name': 'CM.forest'
            }, {
                'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign',
                'type': 'file',
                'name': 'CM.benign'
            }, {
                'path': '/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes',
                'type': 'file',
                'name': 'CM.dunes'
            }],
            'name': 'CRUDE',
            'path': '/generic_root/site_store/MIPK/DORAS/CRUDE'
        }, {
            'type': 'dir',
            'data': [{
                'type': 'dir',
                'data': [{
                    'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant',
                    'type': 'file',
                    'name': 'CM.tangeant'
                }, {
                    'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral',
                    'type': 'file',
                    'name': 'CM.astral'
                }, {
                    'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking',
                    'type': 'file',
                    'name': 'CM.forking'
                }],
                'name': 'CRUDE',
                'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE'
            }],
            'name': 'COMMODITIES',
            'path': '/generic_root/site_store/MIPK/DORAS/COMMODITIES'
        }],
        'name': 'DORAS',
        'path': '/generic_root/site_store/MIPK/DORAS'
    }
}

2个问题：

1. 如何输出正确的格式？（示例如下）
2. 在上面给出了我如何能在时间上进一步优化数以百万计的列表条目的功能？你知道吗

 desired output = {
    "type": "dir",
    "data": [{
        "type": "dir",
        "data": [{
            "path": "/generic_root/site_store/MIPK/CM.toroidal",
            "type": "file",
            "name": "CM.toroidal"
        }, {
            "path": "/generic_root/site_store/MIPK/CM.Supervoid",
            "type": "file",
            "name": "CM.Supervoid"
        }, {
            "type": "dir",
            "data": [{
                "type": "dir",
                "data": [{
                    "path": "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest",
                    "type": "file",
                    "name": "CM.forest"
                }, {
                    "path": "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign",
                    "type": "file",
                    "name": "CM.benign"
                }, {
                    "path": "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes",
                    "type": "file",
                    "name": "CM.dunes"
                }],
                "name": "CRUDE",
                "path": "/generic_root/site_store/MIPK/DORAS/CRUDE"
            }, {
                "type": "dir",
                "data": [{
                    "type": "dir",
                    "data": [{
                        "path": "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant",
                        "type": "file",
                        "name": "CM.tangeant"
                    }, {
                        "path": "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral",
                        "type": "file",
                        "name": "CM.astral"
                    }, {
                        "path": "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking",
                        "type": "file",
                        "name": "CM.forking"
                    }],
                    "name": "CRUDE",
                    "path": "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE"
                }],
                "name": "COMMODITIES",
                "path": "/generic_root/site_store/MIPK/DORAS/COMMODITIES"
            }],
            "name": "DORAS",
            "path": "/generic_root/site_store/MIPK/DORAS"
        }],
        "name": "MIPK",
        "path": "/generic_root/site_store/MIPK"
    }],
    "name": "site_store",
    "path": "/generic_root/site_store"
}

Answer 1

提出了这个解决方案，它使用一个navigator对象在dict上来回走动

首先解析字符串，从路径中拆分成子目录列表。你知道吗

检查它是否存在于当前结果级别（“data”字段中“name”为lamdba处理的blah的项），如果不存在，则创建它并将导航器移到此级别。你知道吗

导航器创建或导航到正确的子目录（父目录列表中的最后一个子目录）后，将节点对象附加到结果数组。你知道吗

对于下一个路径，导航器被重置为完整结果，并且过程再次开始。你知道吗

欢迎反馈，我相信可以改进。你知道吗

def main():
    # Initialise
    root_path    = '/generic_root'
    store        = '/site_store'
    file_list    = [
        "/generic_root/site_store/MIPK/CM.toroidal",
        "/generic_root/site_store/MIPK/CM.Supervoid",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.forest",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.benign",
        "/generic_root/site_store/MIPK/DORAS/CRUDE/CM.dunes",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.tangeant",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.astral",
        "/generic_root/site_store/MIPK/DORAS/COMMODITIES/CRUDE/CM.forking"
    ]
    # Call loop and display results
    viewstore_tree_map = create_dir_tree(file_list, root_path, store)


def Node(full_path, name):
    '''
    full_path[STR] 
    name [STR]
    returns [DICT] node object
    '''
    return {
         'path': full_path,
         'name': name,
         'type': 'dir',
         'data': [],
    }

def Record(full_path):
    '''
    full_path[STR] 
    returns [DICT] record object
    '''
    return {
        'path': full_path,
        'name': full_path.rsplit('/', 1)[-1],
        'type': 'file',
    }

def joiner(delimiter, *args):
    '''
    delimiter[STR] 
    *args [STR]'s unpack sets and lists
    returns [STR] Joins path strings correctly
    '''
    return delimiter.join(list(args))


def create_dir_tree(file_list, root_path, store):
    '''
    file_list [LIST][STR]
    root_path [STR] - root directory
    store [STR] - store site (sub directory)
    returns [DICT] Nested dict of paths from a flat list
    '''
    #  Intiliase
    full_root = self.base_dir + store
    result = Node(full_root, store)
    for path in file_list):
        short_path = path.replace(full_root, '')
        # 1st pass: set navigator to result, 
        # Subsequent: Resets navigator object, and node_path for next path
        navigator = result
        node_path = full_root
        # Remove non-relevant parts of path
        parents = short_path.replace(str_filter, '').split('/')[1:-1]
        for parent in parents:
            node_path = joiner('/', node_path, parent)
            if not filter(lambda dir_rec: dir_rec['name'] == parent,
                          navigator['data']):
                # If node does not exist, append dir object to 'data'
                navigator['data'].append(Node(node_path, parent))
            # Move navigator to previously found node
            navigator = navigator['data'][-1]
        # Append record after constructing or navigating to correct path
        navigator['data'].append(Record(path))
    return result