Upload folder using huggingface_hub

README.md

@@ -1,17 +1,29 @@
 ---
-license: other
+license: apache-2.0
+# TODO 什么License？
 language:
 - en
 - zh
+# TODO 明确支持的语言
 pipeline_tag: audio-text-to-text
 tags:
 - multimodal
 - audio-language-model
 - audio
+# - audio-captioning
+# - audio-classification
+# - audio-generation
+# - audio-question-answering
+# - audio-understanding
+# - chat
+# - speech-recognition
+# - text-to-speech
+# TODO 有什么能力
 base_model:
 - mispeech/dasheng-0.6B
 - Qwen/Qwen2.5-Omni-3B
 base_model_relation: finetune
+# TODO 检查是否正确
 ---
 
 # MiDashengLM
@@ -118,4 +130,4 @@ base_model_relation: finetune
 
 ```bibtex
 TODO
-```
+```

chat_template.jinja

@@ -1,6 +1,6 @@
 {%- for message in messages -%}
 	{%- if loop.first and message["role"] != "system" -%}
-		{{- "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" -}}
+		{{- "<|im_start|>system\nYou are a helpful language and speech assistant.<|im_end|>\n" -}}
 	{%- endif -%}
 	{{- "<|im_start|>" -}}
 	{{- message["role"] -}}

config.json

@@ -36,7 +36,6 @@
     "AutoConfig": "configuration_midashenglm.MiDashengLMConfig",
     "AutoModelForCausalLM": "modeling_midashenglm.MiDashengLMModel"
   },
-  "lora_target_modules": "all-linear",
   "model_type": "midashenglm",
   "subsample_factor": 5,
   "text_config": {

modeling_midashenglm.py

@@ -474,44 +474,22 @@ class MiDashengLMModel(PreTrainedModel):
 
         return encoder_out
 
-    def _prepare_with_input_ids(
+    def _prepare_inputs_embeds(
         self,
-        input_ids: torch.Tensor,
-        audio_embeddings: Optional[torch.Tensor],
-        audio_token_id: Optional[int],
-    ) -> torch.Tensor:
-        input_embeddings = self.decoder.model.embed_tokens(input_ids)
-        if audio_embeddings is not None:
-            special_mask = input_ids == audio_token_id
-            assert audio_embeddings.shape[1] <= (special_mask.sum(-1)).max(), (
-                "Mask and audio embeddings seem to have different sizes: "
-                f"{audio_embeddings.shape=}, {special_mask=}, {input_ids=}, "
-                f"{audio_embeddings.shape[1]=} vs {(special_mask.sum(-1)).max()=}"
-            )
-            audio_embeddings = audio_embeddings.to(input_embeddings.dtype)
-
-            for i in range(len(special_mask)):
-                mask = special_mask[i]
-                number_of_tokens = mask.sum(-1)
-                input_embeddings[i, mask] = audio_embeddings[i, :number_of_tokens]
-            return input_embeddings
-        else:
-            return input_embeddings
-
-    def forward(
-        self,
-        input_ids: Optional[Tensor] = None,
-        input_values: Optional[Tensor] = None,
-        inputs_embeds: Optional[Tensor] = None,
+        input_ids: Optional[torch.Tensor],
+        input_values: Optional[torch.Tensor],
+        inputs_embeds: Optional[torch.Tensor],
         audio_length: Optional[Iterable[int]] = None,
         audio_token_id: Optional[int] = None,
-        **kwargs: Any,
-    ):
+    ) -> torch.Tensor:
         if input_ids is not None:
             if inputs_embeds is not None:
                 raise ValueError(
                     "Both `inputs_embeds` and `input_ids` are passed. Please pass only one of them."
                 )
+            inputs_embeds = cast(
+                torch.Tensor, self.decoder.model.embed_tokens(input_ids)
+            )
 
             if input_values is not None:
                 if audio_token_id is None:
@@ -519,25 +497,31 @@ class MiDashengLMModel(PreTrainedModel):
                         "If `input_values` is provided, `audio_token_id` must also be provided."
                     )
 
-                input_values = input_values.to(self.device)
-                audio_encoder_hidden_states = self._forward_audio_encoder(
-                    input_values, audio_length=audio_length
-                )
-            else:
-                batch, _ = input_ids.shape
-                input_values = torch.zeros(
-                    batch,
-                    0,
-                    self.audio_encoder.embed_dim,
-                    device=input_ids.device,
+                audio_embeddings = self._forward_audio_encoder(
+                    input_values,
+                    audio_length=audio_length,
+                ).to(inputs_embeds.dtype)
+
+                audio_mask = (input_ids == audio_token_id).flatten()
+                diff = torch.diff(
+                    audio_mask.long(),
+                    prepend=torch.zeros(
+                        (1,),
+                        dtype=torch.long,
+                        device=audio_mask.device,
+                    ),
                 )
-
-            input_ids = input_ids.to(self.device)
-            inputs_embeds = self._prepare_with_input_ids(
-                input_ids=input_ids,
-                audio_embeddings=audio_encoder_hidden_states,
-                audio_token_id=audio_token_id,
-            )
+                audio_span_starts = (diff == 1).nonzero()
+                audio_span_ends = (diff == -1).nonzero()
+
+                embeds_view = inputs_embeds.view(-1, inputs_embeds.shape[-1])
+                for span_start, span_end, audio in zip(
+                    audio_span_starts,
+                    audio_span_ends,
+                    audio_embeddings,
+                    strict=True,
+                ):
+                    embeds_view[span_start:span_end] = audio[: span_end - span_start]
         else:
             if inputs_embeds is None:
                 raise ValueError(
@@ -548,6 +532,24 @@ class MiDashengLMModel(PreTrainedModel):
                     "Cannot pass `input_values` when `inputs_embeds` is provided."
                 )
 
+        return inputs_embeds
+
+    def forward(
+        self,
+        input_ids: Optional[Tensor] = None,
+        input_values: Optional[Tensor] = None,
+        inputs_embeds: Optional[Tensor] = None,
+        audio_length: Optional[Iterable[int]] = None,
+        audio_token_id: Optional[int] = None,
+        **kwargs: Any,
+    ):
+        inputs_embeds = self._prepare_inputs_embeds(
+            input_ids=input_ids,
+            input_values=input_values,
+            inputs_embeds=inputs_embeds,
+            audio_length=audio_length,
+            audio_token_id=audio_token_id,
+        )
         return self.decoder(
             input_ids=None,
             inputs_embeds=inputs_embeds,
@@ -563,35 +565,13 @@ class MiDashengLMModel(PreTrainedModel):
         audio_token_id: Optional[int] = None,
         **kwargs,
     ):
-        if input_ids is not None:
-            if inputs_embeds is not None:
-                raise ValueError(
-                    "Both `inputs_embeds` and `input_ids` are passed. Please pass only one of them."
-                )
-            input_ids = input_ids.to(self.device)
-
-            if input_values is not None:
-                input_values = input_values.to(self.device)
-                audio_encoder_hidden_states = self._forward_audio_encoder(
-                    input_values, audio_length=audio_length
-                )
-            else:
-                audio_encoder_hidden_states = None
-            inputs_embeds = self._prepare_with_input_ids(
-                input_ids=input_ids,
-                audio_embeddings=audio_encoder_hidden_states,
-                audio_token_id=audio_token_id,
-            )
-        else:
-            if inputs_embeds is None:
-                raise ValueError(
-                    "Either `input_ids` or `inputs_embeds` must be passed."
-                )
-            if input_values is not None:
-                raise ValueError(
-                    "Cannot pass `input_values` when `inputs_embeds` is provided."
-                )
-
+        inputs_embeds = self._prepare_inputs_embeds(
+            input_ids=input_ids,
+            input_values=input_values,
+            inputs_embeds=inputs_embeds,
+            audio_length=audio_length,
+            audio_token_id=audio_token_id,
+        )
         return self.decoder.generate(
             inputs_embeds=inputs_embeds,
             generation_config=kwargs.pop("generation_config", self.generation_config),