mirror of
https://www.modelscope.cn/deepseek-ai/Janus-Pro-1B.git
synced 2025-04-18 15:49:31 +08:00
Update README.md
This commit is contained in:
ai-modelscope
parent
7409f9fd00
commit
2fd836a168
33
.gitattributes
vendored
33
.gitattributes
vendored
@ -1,47 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
95
README.md
95
README.md
@ -1,47 +1,60 @@
|
||||
---
|
||||
license: Apache License 2.0
|
||||
|
||||
#model-type:
|
||||
##如 gpt、phi、llama、chatglm、baichuan 等
|
||||
#- gpt
|
||||
|
||||
#domain:
|
||||
##如 nlp、cv、audio、multi-modal
|
||||
#- nlp
|
||||
|
||||
#language:
|
||||
##语言代码列表 https://help.aliyun.com/document_detail/215387.html?spm=a2c4g.11186623.0.0.9f8d7467kni6Aa
|
||||
#- cn
|
||||
|
||||
#metrics:
|
||||
##如 CIDEr、Blue、ROUGE 等
|
||||
#- CIDEr
|
||||
|
||||
#tags:
|
||||
##各种自定义，包括 pretrained、fine-tuned、instruction-tuned、RL-tuned 等训练方法和其他
|
||||
#- pretrained
|
||||
|
||||
#tools:
|
||||
##如 vllm、fastchat、llamacpp、AdaSeq 等
|
||||
#- vllm
|
||||
license: mit
|
||||
license_name: deepseek
|
||||
license_link: LICENSE
|
||||
pipeline_tag: any-to-any
|
||||
library_name: transformers
|
||||
tags:
|
||||
- muiltimodal
|
||||
- text-to-image
|
||||
- unified-model
|
||||
---
|
||||
### 当前模型的贡献者未提供更加详细的模型介绍。模型文件和权重，可浏览“模型文件”页面获取。
|
||||
#### 您可以通过如下git clone命令，或者ModelScope SDK来下载模型
|
||||
|
||||
SDK下载
|
||||
```bash
|
||||
#安装ModelScope
|
||||
pip install modelscope
|
||||
## 1. Introduction
|
||||
|
||||
Janus-Pro is a novel autoregressive framework that unifies multimodal understanding and generation.
|
||||
It addresses the limitations of previous approaches by decoupling visual encoding into separate pathways, while still utilizing a single, unified transformer architecture for processing. The decoupling not only alleviates the conflict between the visual encoder’s roles in understanding and generation, but also enhances the framework’s flexibility.
|
||||
Janus-Pro surpasses previous unified model and matches or exceeds the performance of task-specific models.
|
||||
The simplicity, high flexibility, and effectiveness of Janus-Pro make it a strong candidate for next-generation unified multimodal models.
|
||||
|
||||
[**Github Repository**](https://github.com/deepseek-ai/Janus)
|
||||
|
||||
<div align="center">
|
||||
<img alt="image" src="janus_pro_teaser1.png" style="width:90%;">
|
||||
</div>
|
||||
|
||||
<div align="center">
|
||||
<img alt="image" src="janus_pro_teaser2.png" style="width:90%;">
|
||||
</div>
|
||||
|
||||
|
||||
### 2. Model Summary
|
||||
|
||||
Janus-Pro is a unified understanding and generation MLLM, which decouples visual encoding for multimodal understanding and generation.
|
||||
Janus-Pro is constructed based on the DeepSeek-LLM-1.5b-base/DeepSeek-LLM-7b-base.
|
||||
|
||||
For multimodal understanding, it uses the [SigLIP-L](https://huggingface.co/timm/ViT-L-16-SigLIP-384) as the vision encoder, which supports 384 x 384 image input. For image generation, Janus-Pro uses the tokenizer from [here](https://github.com/FoundationVision/LlamaGen) with a downsample rate of 16.
|
||||
|
||||
|
||||
|
||||
## 3. Quick Start
|
||||
|
||||
Please refer to [**Github Repository**](https://github.com/deepseek-ai/Janus)
|
||||
|
||||
|
||||
## 4. License
|
||||
|
||||
This code repository is licensed under [the MIT License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-CODE). The use of Janus-Pro models is subject to [DeepSeek Model License](https://github.com/deepseek-ai/DeepSeek-LLM/blob/HEAD/LICENSE-MODEL).
|
||||
## 5. Citation
|
||||
|
||||
```
|
||||
```python
|
||||
#SDK模型下载
|
||||
from modelscope import snapshot_download
|
||||
model_dir = snapshot_download('deepseek-ai/Janus-Pro-1B')
|
||||
```
|
||||
Git下载
|
||||
```
|
||||
#Git模型下载
|
||||
git clone https://www.modelscope.cn/deepseek-ai/Janus-Pro-1B.git
|
||||
@misc{chen2025januspro,
|
||||
title={Janus-Pro: Unified Multimodal Understanding and Generation with Data and Model Scaling},
|
||||
author={Xiaokang Chen and Zhiyu Wu and Xingchao Liu and Zizheng Pan and Wen Liu and Zhenda Xie and Xingkai Yu and Chong Ruan},
|
||||
year={2025},
|
||||
}
|
||||
```
|
||||
|
||||
<p style="color: lightgrey;">如果您是本模型的贡献者，我们邀请您根据<a href="https://modelscope.cn/docs/ModelScope%E6%A8%A1%E5%9E%8B%E6%8E%A5%E5%85%A5%E6%B5%81%E7%A8%8B%E6%A6%82%E8%A7%88" style="color: lightgrey; text-decoration: underline;">模型贡献文档</a>，及时完善模型卡片内容。</p>
|
||||
## 6. Contact
|
||||
|
||||
If you have any questions, please raise an issue or contact us at [service@deepseek.com](mailto:service@deepseek.com).
|
||||
66
config.json
Normal file
66
config.json
Normal file
@ -0,0 +1,66 @@
|
||||
{
|
||||
"aligner_config": {
|
||||
"cls": "MlpProjector",
|
||||
"model_type": "aligner",
|
||||
"params": {
|
||||
"depth": 2,
|
||||
"input_dim": 1024,
|
||||
"n_embed": 2048,
|
||||
"projector_type": "mlp_gelu"
|
||||
}
|
||||
},
|
||||
"architectures": [
|
||||
"MultiModalityCausalLM"
|
||||
],
|
||||
"gen_aligner_config": {
|
||||
"cls": "MlpProjector",
|
||||
"model_type": "gen_aligner",
|
||||
"params": {
|
||||
"depth": 2,
|
||||
"input_dim": 8,
|
||||
"n_embed": 2048,
|
||||
"projector_type": "mlp_gelu"
|
||||
}
|
||||
},
|
||||
"gen_head_config": {
|
||||
"cls": "vision_head",
|
||||
"model_type": "gen_head",
|
||||
"params": {
|
||||
"image_token_embed": 2048,
|
||||
"image_token_size": 16384,
|
||||
"n_embed": 2048
|
||||
}
|
||||
},
|
||||
"gen_vision_config": {
|
||||
"cls": "VQ-16",
|
||||
"model_type": "gen_vision",
|
||||
"params": {
|
||||
"image_token_size": 16384,
|
||||
"n_embed": 8
|
||||
}
|
||||
},
|
||||
"language_config": {
|
||||
"hidden_size": 2048,
|
||||
"intermediate_size": 5632,
|
||||
"max_position_embeddings": 16384,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 16,
|
||||
"torch_dtype": "bfloat16",
|
||||
"vocab_size": 102400
|
||||
},
|
||||
"model_type": "multi_modality",
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.33.1",
|
||||
"vision_config": {
|
||||
"cls": "CLIPVisionTower",
|
||||
"model_type": "vision",
|
||||
"params": {
|
||||
"image_size": 384,
|
||||
"model_name": "siglip_large_patch16_384",
|
||||
"select_feature": "same",
|
||||
"select_layer": -1
|
||||
}
|
||||
}
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "any-to-any", "allow_remote": true}
|
||||
BIN
janus_pro_teaser1.png
Normal file
BIN
janus_pro_teaser1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 96 KiB |
BIN
janus_pro_teaser2.png
Normal file
BIN
janus_pro_teaser2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 518 KiB |
23
preprocessor_config.json
Normal file
23
preprocessor_config.json
Normal file
@ -0,0 +1,23 @@
|
||||
{
|
||||
"background_color": [
|
||||
127,
|
||||
127,
|
||||
127
|
||||
],
|
||||
"do_normalize": true,
|
||||
"image_mean": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"image_processor_type": "VLMImageProcessor",
|
||||
"image_size": 384,
|
||||
"image_std": [
|
||||
0.5,
|
||||
0.5,
|
||||
0.5
|
||||
],
|
||||
"min_size": 14,
|
||||
"processor_class": "VLChatProcessor",
|
||||
"rescale_factor": 0.00392156862745098
|
||||
}
|
||||
9
processor_config.json
Normal file
9
processor_config.json
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"add_special_token": false,
|
||||
"ignore_id": -100,
|
||||
"image_tag": "<image_placeholder>",
|
||||
"mask_prompt": true,
|
||||
"num_image_tokens": 576,
|
||||
"processor_class": "VLChatProcessor",
|
||||
"sft_format": "deepseek"
|
||||
}
|
||||
BIN
pytorch_model.bin
(Stored with Git LFS)
Normal file
BIN
pytorch_model.bin
(Stored with Git LFS)
Normal file
Binary file not shown.
16
special_tokens_map.json
Normal file
16
special_tokens_map.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<image_placeholder>",
|
||||
"<patch_placeholder>",
|
||||
"<|ref|>",
|
||||
"<|/ref|>",
|
||||
"<|det|>",
|
||||
"<|/det|>",
|
||||
"<|grounding|>",
|
||||
"<|User|>",
|
||||
"<|Assistant|>"
|
||||
],
|
||||
"bos_token": "<｜begin▁of▁sentence｜>",
|
||||
"eos_token": "<｜end▁of▁sentence｜>",
|
||||
"pad_token": "<｜▁pad▁｜>"
|
||||
}
|
||||
205206
tokenizer.json
Normal file
205206
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
10
tokenizer_config.json
Normal file
10
tokenizer_config.json
Normal file
@ -0,0 +1,10 @@
|
||||
{
|
||||
"bos_token": "<｜begin▁of▁sentence｜>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<｜end▁of▁sentence｜>",
|
||||
"model_max_length": 16384,
|
||||
"pad_token": null,
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": null,
|
||||
"use_default_system_prompt": true
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user