解析文件时我的JavaScript代码出现问题
我现在正在做一个项目,目的是用JavaScript代码来读取和解析一些由Python的numpy生成的.npy文件,这些文件的具体信息在一个JSON文件里。这些.npy文件里面存储的是浮点数的数组(也就是机器学习模型的权重和偏置)。不过,我在运行这个脚本的时候遇到了一些错误。我的实现中出现问题的原因是什么呢?该怎么解决这些问题呢?
这是我目前写的代码:
import fs from 'fs';
import path from 'path';
import numpy from 'numpy';
const jsonPath = 'model_weights.json';
const jsonData = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
for (const [layerName, layerData] of Object.entries(jsonData)) {
console.log(`Layer: ${layerName}`);
// Read and print the weight data
const weightFile = layerData.weight;
const weightData = numpy.load(weightFile);
console.log('Weight:');
console.log(weightData);
// Read and print the bias data
const biasFile = layerData.bias;
const biasData = numpy.load(biasFile);
console.log('Bias:');
console.log(biasData);
console.log();
}
这是一个示例JSON文件:
{
"conv_mid.0": {
"weight": "conv_mid.0_weight.npy",
"bias": "conv_mid.0_bias.npy"
},
"conv_mid.1": {
"weight": "conv_mid.1_weight.npy",
"bias": "conv_mid.1_bias.npy"
},
"conv_mid.2": {
"weight": "conv_mid.2_weight.npy",
"bias": "conv_mid.2_bias.npy"
},
"conv_mid.3": {
"weight": "conv_mid.3_weight.npy",
"bias": "conv_mid.3_bias.npy"
}
}
这是我遇到的错误:
node:internal/modules/esm/resolve:214
const resolvedOption = FSLegacyMainResolve(packageJsonUrlString, packageConfig.main, baseStringified);
^
Error: Cannot find package 'node_modules\numpy\package.json' imported from retrieve.mjs
为了让大家更明白,这里是我生成.npy文件的方式:
import torch
import json
import numpy as np
state_dict = torch.load('network.pth')
# Extract the 'params' OrderedDict from the state_dict
params_dict = state_dict['params']
indices = {}
for name, param in params_dict.items():
layer_name, param_type = name.rsplit('.', 1)
if layer_name not in indices:
indices[layer_name] = {}
if param_type == 'weight':
# Store the weight data as a numpy array
data = np.array(param.tolist(), dtype=np.float32)
np.save(f"{layer_name}_weight.npy", data)
indices[layer_name]['weight'] = f"{layer_name}_weight.npy"
elif param_type == 'bias':
# Store the bias data as a numpy array
data = np.array(param.tolist(), dtype=np.float32)
np.save(f"{layer_name}_bias.npy", data)
indices[layer_name]['bias'] = f"{layer_name}_bias.npy"
json_data = json.dumps(indices, indent=4)
with open('model_weights.json', 'w') as f:
f.write(json_data)
print("JSON data and weights/biases saved to model_weights.json and .npy files")
print(json_data)
更新:当我打印loadedData
时,输出是这样的:
{
'conv_head.bias': [
-0.00030437775421887636,
0.0042824335396289825,
0.0022352728992700577,
-0.0019111455185338855,
0.00017375686729792506,
0.0017428217688575387,
0.005364884156733751,
0.0028239202219992876
],
'conv_mid.1.bias': [
-0.002030380303040147,
0.009842530824244022,
-0.0008345193928107619,
-0.007043828722089529,
-0.00633968273177743,
-0.006188599392771721,
-0.0018206692766398191,
0.0037471423856914043
],
'conv_head.weight': [
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ],
[ [Array], [Array], [Array] ]
],
'conv_mid.0.weight': [
[
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array]
],
[
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array], [Array], [Array],
[Array]
], ...
这里是一个可以正常工作的Python版本,用来读取这些文件:
import numpy as np
import json
# Load the JSON data containing the file names and indices
with open('model_weights.json', 'r') as f:
json_data = json.load(f)
# Iterate over the layers
for layer_name, layer_data in json_data.items():
print(f"Layer: {layer_name}")
# Read and print the weight data
weight_file = layer_data['weight']
weight_data = np.load(weight_file)
print("Weight:")
print(weight_data)
# Read and print the bias data
bias_file = layer_data['bias']
bias_data = np.load(bias_file)
print("Bias:")
print(bias_data)
print()
1 个回答
暂无回答