解析文件时我的JavaScript代码出现问题

-1 投票
1 回答
59 浏览
提问于 2025-04-12 03:30

我现在正在做一个项目,目的是用JavaScript代码来读取和解析一些由Python的numpy生成的.npy文件,这些文件的具体信息在一个JSON文件里。这些.npy文件里面存储的是浮点数的数组(也就是机器学习模型的权重和偏置)。不过,我在运行这个脚本的时候遇到了一些错误。我的实现中出现问题的原因是什么呢?该怎么解决这些问题呢?

这是我目前写的代码:

import fs from 'fs';
import path from 'path';
import numpy from 'numpy';

const jsonPath = 'model_weights.json';
const jsonData = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));

for (const [layerName, layerData] of Object.entries(jsonData)) {
  console.log(`Layer: ${layerName}`);

  // Read and print the weight data
  const weightFile = layerData.weight;
  const weightData = numpy.load(weightFile);
  console.log('Weight:');
  console.log(weightData);

  // Read and print the bias data
  const biasFile = layerData.bias;
  const biasData = numpy.load(biasFile);
  console.log('Bias:');
  console.log(biasData);

  console.log();
}

这是一个示例JSON文件:

{
    "conv_mid.0": {
        "weight": "conv_mid.0_weight.npy",
        "bias": "conv_mid.0_bias.npy"
    },
    "conv_mid.1": {
        "weight": "conv_mid.1_weight.npy",
        "bias": "conv_mid.1_bias.npy"
    },
    "conv_mid.2": {
        "weight": "conv_mid.2_weight.npy",
        "bias": "conv_mid.2_bias.npy"
    },
    "conv_mid.3": {
        "weight": "conv_mid.3_weight.npy",
        "bias": "conv_mid.3_bias.npy"
    }
}

这是我遇到的错误:

node:internal/modules/esm/resolve:214
  const resolvedOption = FSLegacyMainResolve(packageJsonUrlString, packageConfig.main, baseStringified);
                         ^

Error: Cannot find package 'node_modules\numpy\package.json' imported from retrieve.mjs

为了让大家更明白,这里是我生成.npy文件的方式:

import torch
import json
import numpy as np

state_dict = torch.load('network.pth')

# Extract the 'params' OrderedDict from the state_dict
params_dict = state_dict['params']

indices = {}

for name, param in params_dict.items():
    layer_name, param_type = name.rsplit('.', 1)

    if layer_name not in indices:
        indices[layer_name] = {}

    if param_type == 'weight':
        # Store the weight data as a numpy array
        data = np.array(param.tolist(), dtype=np.float32)
        np.save(f"{layer_name}_weight.npy", data)
        indices[layer_name]['weight'] = f"{layer_name}_weight.npy"
    elif param_type == 'bias':
        # Store the bias data as a numpy array
        data = np.array(param.tolist(), dtype=np.float32)
        np.save(f"{layer_name}_bias.npy", data)
        indices[layer_name]['bias'] = f"{layer_name}_bias.npy"

json_data = json.dumps(indices, indent=4)

with open('model_weights.json', 'w') as f:
    f.write(json_data)

print("JSON data and weights/biases saved to model_weights.json and .npy files")
print(json_data)

更新:当我打印loadedData时,输出是这样的:

{
  'conv_head.bias': [
    -0.00030437775421887636,
    0.0042824335396289825,
    0.0022352728992700577,
    -0.0019111455185338855,
    0.00017375686729792506,
    0.0017428217688575387,
    0.005364884156733751,
    0.0028239202219992876
  ],
  'conv_mid.1.bias': [
    -0.002030380303040147,
    0.009842530824244022,
    -0.0008345193928107619,
    -0.007043828722089529,
    -0.00633968273177743,
    -0.006188599392771721,
    -0.0018206692766398191,
    0.0037471423856914043
  ],
  'conv_head.weight': [
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ],
    [ [Array], [Array], [Array] ]
  ],
  'conv_mid.0.weight': [
    [
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array]
    ],
    [
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array], [Array], [Array],
      [Array]
    ], ...

这里是一个可以正常工作的Python版本,用来读取这些文件:

import numpy as np
import json

# Load the JSON data containing the file names and indices
with open('model_weights.json', 'r') as f:
    json_data = json.load(f)

# Iterate over the layers
for layer_name, layer_data in json_data.items():
    print(f"Layer: {layer_name}")

    # Read and print the weight data
    weight_file = layer_data['weight']
    weight_data = np.load(weight_file)
    print("Weight:")
    print(weight_data)

    # Read and print the bias data
    bias_file = layer_data['bias']
    bias_data = np.load(bias_file)
    print("Bias:")
    print(bias_data)

    print()

1 个回答

暂无回答

撰写回答