分类任务报错 RuntimeError: 'weight' must be 2-D

[enterdawn]

代码如下：

model=AutoModelForSequenceClassification.from_pretrained("/root/autodl-fs/models/glm-4-voice-9b", num_labels=4,id2label=id2label,device_map=None, label2id=label2id,trust_remote_code=True).cuda()
metric = evaluate.load("./accuracy.py")
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)
training_args = TrainingArguments(
    output_dir="my_class_model",
    learning_rate=2e-5,
    per_device_train_batch_size=2,
    per_device_eval_batch_size=4,
    num_train_epochs=2,
    weight_decay=0.01,
    eval_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    push_to_hub=False,
)
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_train_dataset,
    eval_dataset=tokenized_test_dataset,
    tokenizer=glm_tokenizer,
    compute_metrics=compute_metrics,
)
trainer.train()

错误信息如下：

RuntimeError                              Traceback (most recent call last)
Cell In[24], line 1
----> 1 trainer.train()

File [~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py:1938](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py#line=1937), in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1936         hf_hub_utils.enable_progress_bars()
   1937 else:
-> 1938     return inner_training_loop(
   1939         args=args,
   1940         resume_from_checkpoint=resume_from_checkpoint,
   1941         trial=trial,
   1942         ignore_keys_for_eval=ignore_keys_for_eval,
   1943     )

File [~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py:2279](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py#line=2278), in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   2276     self.control = self.callback_handler.on_step_begin(args, self.state, self.control)
   2278 with self.accelerator.accumulate(model):
-> 2279     tr_loss_step = self.training_step(model, inputs)
   2281 if (
   2282     args.logging_nan_inf_filter
   2283     and not is_torch_xla_available()
   2284     and (torch.isnan(tr_loss_step) or torch.isinf(tr_loss_step))
   2285 ):
   2286     # if loss is nan or inf simply add the average of previous logged losses
   2287     tr_loss += tr_loss / (1 + self.state.global_step - self._globalstep_last_logged)

File [~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py:3318](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py#line=3317), in Trainer.training_step(self, model, inputs)
   3315     return loss_mb.reduce_mean().detach().to(self.args.device)
   3317 with self.compute_loss_context_manager():
-> 3318     loss = self.compute_loss(model, inputs)
   3320 del inputs
   3321 if (
   3322     self.args.torch_empty_cache_steps is not None
   3323     and self.state.global_step % self.args.torch_empty_cache_steps == 0
   3324 ):

File [~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py:3363](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/transformers/trainer.py#line=3362), in Trainer.compute_loss(self, model, inputs, return_outputs)
   3361 else:
   3362     labels = None
-> 3363 outputs = model(**inputs)
   3364 # Save past state if it exists
   3365 # TODO: this needs to be fixed and made cleaner later.
   3366 if self.args.past_index >= 0:

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1736](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1735), in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1747](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1746), in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File [~/miniconda3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py:18](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/deepspeed/utils/nvtx.py#line=17), in instrument_w_nvtx.<locals>.wrapped_fn(*args, **kwargs)
     16 if enable_nvtx:
     17     get_accelerator().range_push(func.__qualname__)
---> 18 ret_val = func(*args, **kwargs)
     19 if enable_nvtx:
     20     get_accelerator().range_pop()

File [~/miniconda3/lib/python3.10/site-packages/deepspeed/runtime/engine.py:1899](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/deepspeed/runtime/engine.py#line=1898), in DeepSpeedEngine.forward(self, *inputs, **kwargs)
   1896 if self.fp16_auto_cast():
   1897     inputs = self._cast_inputs_half(inputs)
-> 1899 loss = self.module(*inputs, **kwargs)
   1901 if self.zero_optimization_partition_weights():
   1902     # Disable automated discovery of external parameters
   1903     for module in self.module.modules():

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1736](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1735), in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1747](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1746), in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File [~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py:1085](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py#line=1084), in ChatGLMForSequenceClassification.forward(self, input_ids, position_ids, attention_mask, full_attention_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)
   1069 def forward(
   1070         self,
   1071         input_ids: Optional[torch.LongTensor] = None,
   (...)
   1081         return_dict: Optional[bool] = None,
   1082 ) -> Union[Tuple[torch.Tensor, ...], SequenceClassifierOutputWithPast]:
   1083     return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-> 1085     transformer_outputs = self.transformer(
   1086         input_ids=input_ids,
   1087         position_ids=position_ids,
   1088         attention_mask=attention_mask,
   1089         full_attention_mask=full_attention_mask,
   1090         past_key_values=past_key_values,
   1091         inputs_embeds=inputs_embeds,
   1092         use_cache=use_cache,
   1093         output_attentions=output_attentions,
   1094         output_hidden_states=output_hidden_states,
   1095         return_dict=return_dict,
   1096     )
   1098     hidden_states = transformer_outputs[0]
   1099     pooled_hidden_states = hidden_states[:, -1]

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1736](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1735), in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1747](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1746), in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File [~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py:877](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py#line=876), in ChatGLMModel.forward(self, input_ids, position_ids, attention_mask, full_attention_mask, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict)
    874 batch_size, seq_length = input_ids.shape
    876 if inputs_embeds is None:
--> 877     inputs_embeds = self.embedding(input_ids)
    879 if full_attention_mask is None:
    880     if (attention_mask is not None and not attention_mask.all()) or (past_key_values and seq_length != 1):

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1736](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1735), in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1747](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1746), in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File [~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py:813](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/.cache/huggingface/modules/transformers_modules/glm-4-voice-9b/modeling_chatglm.py#line=812), in Embedding.forward(self, input_ids)
    811 def forward(self, input_ids):
    812     # Embeddings.
--> 813     words_embeddings = self.word_embeddings(input_ids)
    814     embeddings = words_embeddings
    815     # If the input flag for fp32 residual connection is set, convert for float.

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1736](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1735), in Module._wrapped_call_impl(self, *args, **kwargs)
   1734     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1735 else:
-> 1736     return self._call_impl(*args, **kwargs)

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py:1747](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/module.py#line=1746), in Module._call_impl(self, *args, **kwargs)
   1742 # If we don't have any hooks, we want to skip the rest of the logic in
   1743 # this function, and just call forward.
   1744 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1745         or _global_backward_pre_hooks or _global_backward_hooks
   1746         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1747     return forward_call(*args, **kwargs)
   1749 result = None
   1750 called_always_called_hooks = set()

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/sparse.py:190](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/modules/sparse.py#line=189), in Embedding.forward(self, input)
    189 def forward(self, input: Tensor) -> Tensor:
--> 190     return F.embedding(
    191         input,
    192         self.weight,
    193         self.padding_idx,
    194         self.max_norm,
    195         self.norm_type,
    196         self.scale_grad_by_freq,
    197         self.sparse,
    198     )

File [~/miniconda3/lib/python3.10/site-packages/torch/nn/functional.py:2551](http://region-3.seetacloud.com:23008/jupyter/lab/tree/autodl-fs/~/miniconda3/lib/python3.10/site-packages/torch/nn/functional.py#line=2550), in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   2545     # Note [embedding_renorm set_grad_enabled]
   2546     # XXX: equivalent to
   2547     # with torch.no_grad():
   2548     #   torch.embedding_renorm_
   2549     # remove once script supports set_grad_enabled
   2550     _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 2551 return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)

RuntimeError: 'weight' must be 2-D

[sixsixcoder]

你给的代码并不完整，看报错可能是输出权重的时候不是2维的？

[enterdawn]

你给的代码并不完整，看报错可能是输出权重的时候不是2维的？

问题已解决，在jupyter中对模型进行二次加载就会导致模型权重全部消失。