Downloads · 30 days
14
6% of all-time downloads
Yosegi-0604
Local-Novel-LLM-project/Yosegi-0604
Yosegi-0604 is a text generation model from Local-Novel-LLM-project. Use it when you need the model to write or continue text. It is set up for transformers.
This is a merge of pre-trained language models created using mergekit.
All-time downloads
240
Public
Parameters
7.2B
14.5 GB on disk
Likes
1
Public
Repo makeup
Click a slice to open those files.
.safetensors14.5 GB · 100%
At a glance
- Task
- Text Generation
- Library
- transformers
- Model type
- mistral
- Access
- Public
- Created
- Jun 22, 2024
- Updated
- Jun 22, 2024
- SHA
- c9aed4ac
Try a prompt
Model card
From the Hugging Face model README
final_model
This is a merge of pre-trained language models created using mergekit.
Merge Details
Merge Method
This model was merged using the breadcrumbs_ties merge method using ./Yosegi-0603 as a base.
Models Merged
The following models were included in the merge:
- ./Ninja-2B_JP
- ./Yosegi-0601
Configuration
The following YAML configuration was used to produce this model:
base_model: ./Yosegi-0603
dtype: bfloat16
merge_method: breadcrumbs_ties
parameters:
int8_mask: 1.0
normalize: 0.0
slices:
- sources:
- layer_range: [0, 2]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.050387850856855765
- filter: mlp
value: -0.17075015661203768
- value: -0.008041653902986862
weight:
- filter: self_attn
value: 0.0999312941470471
- filter: mlp
value: 0.541727762184749
- value: 0.6837012779994258
- layer_range: [0, 2]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.8218846237599902
- filter: mlp
value: 1.0
- value: 0.9254078866667358
gamma:
- filter: self_attn
value: -0.11213758231875963
- filter: mlp
value: 0.021586098873668948
- value: -0.12827998218659437
weight:
- filter: self_attn
value: 0.40391646444657003
- filter: mlp
value: 0.623121864641881
- value: 0.5967833694632534
- layer_range: [0, 2]
model: ./Yosegi-0603
- sources:
- layer_range: [2, 4]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.8079479346300947
- filter: mlp
value: 1.0
- value: 0.710146185559419
gamma:
- filter: self_attn
value: 0.1383609589681566
- filter: mlp
value: 0.21188532059635062
- value: 0.2994723556443468
weight:
- filter: self_attn
value: 0.48107070906079974
- filter: mlp
value: 0.5848073552919492
- value: 0.4583842493359253
- layer_range: [2, 4]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 0.934378153535579
- value: 1.0
gamma:
- filter: self_attn
value: 0.073192612278188
- filter: mlp
value: 0.07939126555063317
- value: -0.06891845030175699
weight:
- filter: self_attn
value: 0.32120386994101
- filter: mlp
value: 0.5001108459121922
- value: 0.9138710221666694
- layer_range: [2, 4]
model: ./Yosegi-0603
- sources:
- layer_range: [4, 6]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 0.7237519222177541
- value: 0.776951124863642
gamma:
- filter: self_attn
value: -0.2265121048274062
- filter: mlp
value: -0.1757947421960496
- value: -0.11401593728931929
weight:
- filter: self_attn
value: 0.6448742737026658
- filter: mlp
value: 0.13809748641457986
- value: 0.3950550285769662
- layer_range: [4, 6]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.9649359194114893
- filter: mlp
value: 0.916637032428399
- value: 1.0
gamma:
- filter: self_attn
value: -0.16291684846287688
- filter: mlp
value: -0.19013548712121703
- value: 0.038409066391918795
weight:
- filter: self_attn
value: 0.1977358472772336
- filter: mlp
value: 0.22661167907612348
- value: 0.6426575016448257
- layer_range: [4, 6]
model: ./Yosegi-0603
- sources:
- layer_range: [6, 8]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.8727809666891416
- filter: mlp
value: 1.0
- value: 0.5160677785559116
gamma:
- filter: self_attn
value: 0.14245180617134273
- filter: mlp
value: 0.08189992601998919
- value: -0.1038827997670827
weight:
- filter: self_attn
value: 0.23575676914257698
- filter: mlp
value: 0.4047231670507743
- value: 0.34207794631274374
- layer_range: [6, 8]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: 0.576775501046583
- filter: mlp
value: -0.046028636298718645
- value: -0.024161321403060265
weight:
- filter: self_attn
value: 0.833089842843994
- filter: mlp
value: 0.5434667434613458
- value: 0.2946693008513797
- layer_range: [6, 8]
model: ./Yosegi-0603
- sources:
- layer_range: [8, 10]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 0.9930269337531187
gamma:
- filter: self_attn
value: 0.4549980941970383
- filter: mlp
value: 0.10362988739411173
- value: -0.43800391668559174
weight:
- filter: self_attn
value: 0.19663450954683193
- filter: mlp
value: 0.16783989984505265
- value: 0.7465091417598162
- layer_range: [8, 10]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.797370597380894
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.0665958634205702
- filter: mlp
value: -0.058297473060129834
- value: -0.38206760673090134
weight:
- filter: self_attn
value: 0.7015967347604024
- filter: mlp
value: 0.7733694864324641
- value: 0.7636921732342238
- layer_range: [8, 10]
model: ./Yosegi-0603
- sources:
- layer_range: [10, 12]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.8047576867589878
- filter: mlp
value: 0.8852533319203653
- value: 0.7707342647603538
gamma:
- filter: self_attn
value: -0.054343999574509694
- filter: mlp
value: -0.3465154355167133
- value: 0.022315854655582765
weight:
- filter: self_attn
value: 0.4396484757291151
- filter: mlp
value: 0.34318396468602314
- value: 0.8236034746664869
- layer_range: [10, 12]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.9058471193805165
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: 0.1221058588826469
- filter: mlp
value: -0.4004985640890659
- value: 0.3219195440395816
weight:
- filter: self_attn
value: 0.3565443612269864
- filter: mlp
value: 0.2817057075232181
- value: 0.5934890337808251
- layer_range: [10, 12]
model: ./Yosegi-0603
- sources:
- layer_range: [12, 14]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.027897116191693133
- filter: mlp
value: -0.1765379388255607
- value: 0.09108936063176161
weight:
- filter: self_attn
value: 0.4499753137521779
- filter: mlp
value: 0.901296236087911
- value: 0.3548680126954006
- layer_range: [12, 14]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.8973815150776497
- filter: mlp
value: 0.6029953465961999
- value: 1.0
gamma:
- filter: self_attn
value: 0.10393082898402586
- filter: mlp
value: 0.15993577688878796
- value: 0.011410411917833683
weight:
- filter: self_attn
value: 0.2211644023056492
- filter: mlp
value: 0.5677387594231849
- value: 0.1316535663010981
- layer_range: [12, 14]
model: ./Yosegi-0603
- sources:
- layer_range: [14, 16]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.9584597245055072
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.17789727632680347
- filter: mlp
value: 0.2182263440314275
- value: 0.1449547656126498
weight:
- filter: self_attn
value: 0.4551004762874224
- filter: mlp
value: 0.9182082826762857
- value: 0.3736989395186422
- layer_range: [14, 16]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.7414465107848625
- filter: mlp
value: 1.0
- value: 0.7894887419395906
gamma:
- filter: self_attn
value: -0.07343933395880992
- filter: mlp
value: 0.250800731630588
- value: -0.2948778134297542
weight:
- filter: self_attn
value: 0.43125199001016495
- filter: mlp
value: 0.6182726353394477
- value: 0.838902157446268
- layer_range: [14, 16]
model: ./Yosegi-0603
- sources:
- layer_range: [16, 18]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.9474287877268394
- filter: mlp
value: 1.0
- value: 0.9613380133344519
gamma:
- filter: self_attn
value: -0.08608895546593046
- filter: mlp
value: -0.07275416053291164
- value: -0.5796137860399382
weight:
- filter: self_attn
value: 0.5593420897751296
- filter: mlp
value: 0.7339447992880666
- value: 0.5447558586689005
- layer_range: [16, 18]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.9321536960575384
- filter: mlp
value: 1.0
- value: 0.9613033408813294
gamma:
- filter: self_attn
value: 0.20610728738224296
- filter: mlp
value: 0.2002206706624053
- value: -0.45349278793293785
weight:
- filter: self_attn
value: 0.16162975594196963
- filter: mlp
value: 0.21262726992327483
- value: 0.061213622827234075
- layer_range: [16, 18]
model: ./Yosegi-0603
- sources:
- layer_range: [18, 20]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: 0.03922456593148313
- filter: mlp
value: 0.3318035822806869
- value: -0.10373990685028205
weight:
- filter: self_attn
value: 0.8254441016674987
- filter: mlp
value: 0.4568039342431161
- value: 0.3152648515747969
- layer_range: [18, 20]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 0.9807358937293073
gamma:
- filter: self_attn
value: -0.22734036563128657
- filter: mlp
value: 0.26113222150270854
- value: 0.17739039022957015
weight:
- filter: self_attn
value: 0.33759130475641996
- filter: mlp
value: 0.616639215544168
- value: 0.47560658618977714
- layer_range: [18, 20]
model: ./Yosegi-0603
- sources:
- layer_range: [20, 22]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.9394514442960196
- filter: mlp
value: 1.0
- value: 0.9885037757465567
gamma:
- filter: self_attn
value: -0.17365709450334324
- filter: mlp
value: 0.0712279381144505
- value: 0.11809665485306464
weight:
- filter: self_attn
value: 0.485610337254665
- filter: mlp
value: 0.8406593173801935
- value: 0.5024102481819739
- layer_range: [20, 22]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.09980202641768818
- filter: mlp
value: 0.051454493742856926
- value: 0.14619126408666103
weight:
- filter: self_attn
value: 0.54772456079406
- filter: mlp
value: 0.3440893571099615
- value: 0.3747271233512448
- layer_range: [20, 22]
model: ./Yosegi-0603
- sources:
- layer_range: [22, 24]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 0.9474712362889293
- value: 1.0
gamma:
- filter: self_attn
value: -0.16020032978118146
- filter: mlp
value: -0.025085248873309034
- value: 0.06046174910893976
weight:
- filter: self_attn
value: 0.8654189362345427
- filter: mlp
value: 0.6344956382288498
- value: 0.6383979001549549
- layer_range: [22, 24]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.8240762427167851
- filter: mlp
value: 1.0
- value: 0.9004913821398048
gamma:
- filter: self_attn
value: -0.12224186789525764
- filter: mlp
value: -0.25877585460700525
- value: 0.35149388360871714
weight:
- filter: self_attn
value: 0.4294356408713786
- filter: mlp
value: 0.3920647298630233
- value: 0.795891295390721
- layer_range: [22, 24]
model: ./Yosegi-0603
- sources:
- layer_range: [24, 26]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: 0.16915580088030202
- filter: mlp
value: 0.2602652727555053
- value: 0.16985672723305376
weight:
- filter: self_attn
value: 0.420377024485687
- filter: mlp
value: 0.3401141209432324
- value: 0.4953511256159331
- layer_range: [24, 26]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.7290652609253236
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.1039167464696765
- filter: mlp
value: -0.18476572570059685
- value: 0.1221387313921081
weight:
- filter: self_attn
value: 0.2925002157134928
- filter: mlp
value: 0.3854740639588027
- value: 0.555448110317977
- layer_range: [24, 26]
model: ./Yosegi-0603
- sources:
- layer_range: [26, 28]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 1.0
- filter: mlp
value: 0.9104496350690235
- value: 1.0
gamma:
- filter: self_attn
value: 0.24831264214235005
- filter: mlp
value: -0.03903149241855605
- value: 0.14189425093398259
weight:
- filter: self_attn
value: 0.7685811138035815
- filter: mlp
value: 0.06535011571274918
- value: 0.696502559577317
- layer_range: [26, 28]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.9236218028490522
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.2451400735890047
- filter: mlp
value: -0.21555851418482214
- value: 0.020418471695148876
weight:
- filter: self_attn
value: 0.451368534421561
- filter: mlp
value: 0.27412879847687055
- value: 0.18339776770537336
- layer_range: [26, 28]
model: ./Yosegi-0603
- sources:
- layer_range: [28, 30]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.8590812961904566
- filter: mlp
value: 1.0
- value: 1.0
gamma:
- filter: self_attn
value: -0.06934549536310654
- filter: mlp
value: -0.28464693250998063
- value: -0.0588491947891552
weight:
- filter: self_attn
value: 0.26716389671655294
- filter: mlp
value: 0.8228280162386532
- value: 0.24197568479527135
- layer_range: [28, 30]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.7277181780542642
- filter: mlp
value: 0.74166025738732
- value: 1.0
gamma:
- filter: self_attn
value: 0.1772650150670655
- filter: mlp
value: 0.06545031487123437
- value: -0.28681451125993446
weight:
- filter: self_attn
value: 0.5781944040541174
- filter: mlp
value: 0.2288692970435767
- value: 0.689751088930503
- layer_range: [28, 30]
model: ./Yosegi-0603
- sources:
- layer_range: [30, 32]
model: ./Yosegi-0601
parameters:
density:
- filter: self_attn
value: 0.8177341862620365
- filter: mlp
value: 0.8875629677599377
- value: 1.0
gamma:
- filter: self_attn
value: -0.06572527259889459
- filter: mlp
value: -0.18979543285938766
- value: -0.24122036571646263
weight:
- filter: self_attn
value: 0.5818433594657613
- filter: mlp
value: 0.36676821100234736
- value: 0.3580688869263428
- layer_range: [30, 32]
model: ./Ninja-2B_JP
parameters:
density:
- filter: self_attn
value: 0.8306036003344672
- filter: mlp
value: 0.6993970248745297
- value: 1.0
gamma:
- filter: self_attn
value: -0.20599853236581384
- filter: mlp
value: -0.2001187634455465
- value: -0.07654635090020837
weight:
- filter: self_attn
value: 0.37120677279712305
- filter: mlp
value: 0.13105486609905853
- value: 0.7204857820148367
- layer_range: [30, 32]
model: ./Yosegi-0603
tokenizer_source: union
- Task
- Text Generation
- Library
- transformers
- Type
- mistral
- Created
- Jun 22, 2024
- Updated
- Jun 22, 2024