mistralai-mixtral-8x7b-_3473

submission_id: mistralai-mixtral-8x7b-_3473_v68

developer_uid: robert_irvine

status: inactive

model_repo: mistralai/Mixtral-8x7B-Instruct-v0.1

reward_repo: ChaiML/gpt2_xl_pairwise_89m_step_347634

generation_params: {'temperature': 0.9, 'top_p': 1.0, 'min_p': 0.0, 'top_k': 50, 'presence_penalty': 0.5, 'frequency_penalty': 0.5, 'stopping_words': ['\n', '</s>', '<|user|>', '###'], 'max_input_tokens': 512, 'best_of': 4, 'max_output_tokens': 64}

formatter: {'memory_template': '<s>[INST] This is an entertaining conversation. You are {bot_name} who has the persona: {memory}.\nEngage in a chat with {user_name} while staying in character. Try to flirt with {user_name}. Engage in *roleplay* actions. Describe the scene dramatically. \n', 'prompt_template': '{prompt}\n', 'bot_template': '{bot_name}: {message}</s>', 'user_template': '[INST] {user_name}: {message} [/INST]', 'response_template': '[INST] respond with humor [/INST]{bot_name}:', 'truncate_by_message': False}

reward_formatter: {'memory_template': '""', 'prompt_template': '""', 'bot_template': 'Bot: {message}\n', 'user_template': 'User: {message}\n', 'response_template': 'Bot:', 'truncate_by_message': False}

timestamp: 2024-07-10T18:22:25+00:00

model_name: mistralai-mixtral-8x7b-_3473_v68

model_group: mistralai/Mixtral-8x7B-I

num_battles: 34807

num_wins: 14920

celo_rating: 1125.27

alignment_score: None

alignment_samples: 0

propriety_score: 0.7368943277658343

propriety_total_count: 6047.0

submission_type: basic

model_architecture: MixtralForCausalLM

model_num_parameters: 46702792704.0

best_of: 4

max_input_tokens: 512

max_output_tokens: 64

display_name: mistralai-mixtral-8x7b-_3473_v68

ineligible_reason: None

language_model: mistralai/Mixtral-8x7B-Instruct-v0.1

model_size: 47B

reward_model: ChaiML/gpt2_xl_pairwise_89m_step_347634

us_pacific_date: 2024-07-10

win_ratio: 0.42864940960151693

preference_data_url: None

Resubmit model

Reference Models

Running pipeline stage MKMLizer
Starting job with name mistralai-mixtral-8x7b-3473-v68-mkmlizer
Waiting for job on mistralai-mixtral-8x7b-3473-v68-mkmlizer to finish
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ╔═════════════════════════════════════════════════════════════════════╗
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║     _____            __           __                                ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║    / _/ /_ ___    __/ /  ___ ___ / /                                ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║   / _/ / // / |/|/ / _ \/ -_) -_) /                                 ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  /_//_/\_, /|__,__/_//_/\__/\__/_/                                  ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║       /___/                                                         ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║                                                                     ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  Version: 0.8.14                                                    ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  Copyright 2023 MK ONE TECHNOLOGIES Inc.                            ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  https://mk1.ai                                                     ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║                                                                     ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  The license key for the current software has been verified as      ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  belonging to:                                                      ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║                                                                     ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  Chai Research Corp.                                                ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  Account ID: 7997a29f-0ceb-4cc7-9adf-840c57b4ae6f                   ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║  Expiration: 2024-10-15 23:59:59                                    ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ║                                                                     ║
mistralai-mixtral-8x7b-3473-v68-mkmlizer: ╚═════════════════════════════════════════════════════════════════════╝
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Downloaded to shared memory in 272.896s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: quantizing model to /dev/shm/model_cache
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Saving flywheel model at /dev/shm/model_cache
Connection pool is full, discarding connection: %s
Connection pool is full, discarding connection: %s
Connection pool is full, discarding connection: %s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: quantized model in 118.091s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Processed model mistralai/Mixtral-8x7B-Instruct-v0.1 in 390.988s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: creating bucket guanaco-mkml-models
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Bucket 's3://guanaco-mkml-models/' created
mistralai-mixtral-8x7b-3473-v68-mkmlizer: uploading /dev/shm/model_cache to s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/special_tokens_map.json s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/special_tokens_map.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/tokenizer_config.json s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/tokenizer_config.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/tokenizer.json s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/tokenizer.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/config.json s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/config.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/tokenizer.model s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/tokenizer.model
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/flywheel_model.3.safetensors s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/flywheel_model.3.safetensors
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/flywheel_model.0.safetensors s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/flywheel_model.0.safetensors
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/flywheel_model.1.safetensors s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/flywheel_model.1.safetensors
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /dev/shm/model_cache/flywheel_model.2.safetensors s3://guanaco-mkml-models/mistralai-mixtral-8x7b-3473-v68/flywheel_model.2.safetensors
mistralai-mixtral-8x7b-3473-v68-mkmlizer: loading reward model from ChaiML/gpt2_xl_pairwise_89m_step_347634
mistralai-mixtral-8x7b-3473-v68-mkmlizer: 
Loading 0:   0%|          | 0/995 [00:00<?, ?it/s]
Loading 0:   1%|          | 7/995 [00:00<00:16, 60.58it/s]
Loading 0:   1%|▏         | 14/995 [00:00<00:17, 55.83it/s]
Loading 0:   2%|▏         | 20/995 [00:00<00:18, 53.22it/s]
Loading 0:   3%|▎         | 26/995 [00:00<00:17, 54.15it/s]
Loading 0:   4%|▎         | 37/995 [00:00<00:13, 70.09it/s]
Loading 0:   5%|▍         | 45/995 [00:00<00:15, 62.94it/s]
Loading 0:   5%|▌         | 52/995 [00:01<00:27, 34.53it/s]
Loading 0:   6%|▌         | 58/995 [00:01<00:24, 38.06it/s]
Loading 0:   7%|▋         | 66/995 [00:01<00:20, 44.61it/s]
Loading 0:   7%|▋         | 72/995 [00:01<00:20, 45.79it/s]
Loading 0:   8%|▊         | 78/995 [00:01<00:19, 47.64it/s]
Loading 0:   8%|▊         | 84/995 [00:01<00:18, 48.91it/s]
Loading 0:  10%|▉         | 96/995 [00:01<00:13, 64.65it/s]
Loading 0:  10%|█         | 103/995 [00:01<00:13, 64.43it/s]
Loading 0:  11%|█         | 110/995 [00:02<00:26, 32.99it/s]
Loading 0:  12%|█▏        | 116/995 [00:02<00:24, 35.76it/s]
Loading 0:  12%|█▏        | 122/995 [00:02<00:22, 38.99it/s]
Loading 0:  13%|█▎        | 129/995 [00:02<00:19, 44.00it/s]
Loading 0:  14%|█▎        | 135/995 [00:02<00:19, 44.74it/s]
Loading 0:  14%|█▍        | 141/995 [00:03<00:18, 45.62it/s]
Loading 0:  15%|█▍        | 147/995 [00:03<00:18, 47.10it/s]
Loading 0:  16%|█▋        | 162/995 [00:03<00:21, 38.86it/s]
Loading 0:  17%|█▋        | 168/995 [00:03<00:20, 41.25it/s]
Loading 0:  17%|█▋        | 174/995 [00:03<00:18, 43.45it/s]
Loading 0:  18%|█▊        | 179/995 [00:03<00:18, 44.20it/s]
Loading 0:  18%|█▊        | 184/995 [00:04<00:17, 45.10it/s]
Loading 0:  19%|█▉        | 192/995 [00:04<00:15, 51.48it/s]
Loading 0:  20%|█▉        | 198/995 [00:04<00:15, 51.61it/s]
Loading 0:  21%|██        | 204/995 [00:04<00:15, 50.59it/s]
Loading 0:  21%|██        | 210/995 [00:04<00:25, 30.21it/s]
Loading 0:  22%|██▏       | 216/995 [00:04<00:22, 34.22it/s]
Loading 0:  22%|██▏       | 223/995 [00:05<00:18, 40.74it/s]
Loading 0:  23%|██▎       | 229/995 [00:05<00:17, 43.46it/s]
Loading 0:  24%|██▎       | 235/995 [00:05<00:16, 45.68it/s]
Loading 0:  24%|██▍       | 241/995 [00:05<00:15, 47.36it/s]
Loading 0:  25%|██▌       | 253/995 [00:05<00:12, 60.83it/s]
Loading 0:  26%|██▌       | 260/995 [00:05<00:12, 59.19it/s]
Loading 0:  27%|██▋       | 267/995 [00:06<00:21, 33.13it/s]
Loading 0:  28%|██▊       | 277/995 [00:06<00:16, 43.06it/s]
Loading 0:  28%|██▊       | 281/995 [00:16<00:16, 43.06it/s]
Loading 0:  28%|██▊       | 282/995 [00:32<13:16,  1.12s/it]
Loading 0:  29%|██▉       | 290/995 [00:32<08:58,  1.31it/s]
Loading 0:  30%|██▉       | 297/995 [00:33<06:24,  1.82it/s]
Loading 0:  30%|███       | 303/995 [00:33<04:45,  2.42it/s]
Loading 0:  31%|███       | 309/995 [00:33<03:30,  3.27it/s]
Loading 0:  32%|███▏      | 320/995 [00:33<02:14,  5.02it/s]
Loading 0:  33%|███▎      | 326/995 [00:33<01:43,  6.45it/s]
Loading 0:  33%|███▎      | 332/995 [00:33<01:19,  8.34it/s]
Loading 0:  34%|███▍      | 337/995 [00:34<01:03, 10.36it/s]
Loading 0:  35%|███▍      | 344/995 [00:34<00:45, 14.19it/s]
Loading 0:  35%|███▌      | 350/995 [00:34<00:36, 17.82it/s]
Loading 0:  36%|███▌      | 356/995 [00:34<00:29, 21.92it/s]
Loading 0:  36%|███▋      | 362/995 [00:34<00:24, 26.32it/s]
Loading 0:  37%|███▋      | 368/995 [00:34<00:28, 21.63it/s]
Loading 0:  38%|███▊      | 376/995 [00:35<00:21, 28.60it/s]
Loading 0:  38%|███▊      | 381/995 [00:35<00:19, 31.67it/s]
Loading 0:  39%|███▉      | 386/995 [00:35<00:17, 34.79it/s]
Loading 0:  39%|███▉      | 392/995 [00:35<00:15, 38.70it/s]
Loading 0:  40%|████      | 402/995 [00:35<00:11, 51.69it/s]
Loading 0:  41%|████      | 409/995 [00:35<00:10, 55.00it/s]
Loading 0:  42%|████▏     | 416/995 [00:35<00:11, 52.49it/s]
Loading 0:  43%|████▎     | 423/995 [00:36<00:18, 31.71it/s]
Loading 0:  43%|████▎     | 428/995 [00:36<00:16, 34.36it/s]
Loading 0:  44%|████▍     | 436/995 [00:36<00:13, 41.56it/s]
Loading 0:  44%|████▍     | 442/995 [00:36<00:12, 43.85it/s]
Loading 0:  45%|████▌     | 448/995 [00:36<00:12, 45.13it/s]
Loading 0:  46%|████▌     | 454/995 [00:36<00:11, 45.51it/s]
Loading 0:  46%|████▋     | 461/995 [00:36<00:10, 51.28it/s]
Loading 0:  47%|████▋     | 471/995 [00:36<00:08, 61.22it/s]
Loading 0:  48%|████▊     | 478/995 [00:37<00:15, 33.89it/s]
Loading 0:  49%|████▊     | 484/995 [00:37<00:13, 36.56it/s]
Loading 0:  49%|████▉     | 489/995 [00:37<00:13, 38.46it/s]
Loading 0:  50%|████▉     | 495/995 [00:37<00:12, 41.53it/s]
Loading 0:  51%|█████     | 503/995 [00:37<00:10, 48.30it/s]
Loading 0:  51%|█████     | 509/995 [00:38<00:09, 48.83it/s]
Loading 0:  52%|█████▏    | 515/995 [00:38<00:09, 48.33it/s]
Loading 0:  52%|█████▏    | 521/995 [00:38<00:09, 51.07it/s]
Loading 0:  53%|█████▎    | 527/995 [00:38<00:15, 29.34it/s]
Loading 0:  54%|█████▎    | 534/995 [00:38<00:13, 35.13it/s]
Loading 0:  54%|█████▍    | 540/995 [00:38<00:11, 38.66it/s]
Loading 0:  55%|█████▍    | 546/995 [00:39<00:10, 41.70it/s]
Loading 0:  55%|█████▌    | 551/995 [00:39<00:10, 43.12it/s]
Loading 0:  56%|█████▋    | 560/995 [00:52<00:10, 43.12it/s]
Loading 0:  56%|█████▋    | 561/995 [01:05<08:16,  1.14s/it]
Loading 0:  57%|█████▋    | 567/995 [01:05<06:01,  1.18it/s]
Loading 0:  57%|█████▋    | 572/995 [01:06<04:33,  1.55it/s]
Loading 0:  58%|█████▊    | 581/995 [01:06<02:55,  2.36it/s]
Loading 0:  59%|█████▉    | 586/995 [01:06<02:14,  3.03it/s]
Loading 0:  60%|█████▉    | 593/995 [01:06<01:32,  4.35it/s]
Loading 0:  60%|██████    | 599/995 [01:06<01:07,  5.85it/s]
Loading 0:  61%|██████    | 604/995 [01:07<00:52,  7.50it/s]
Loading 0:  61%|██████    | 609/995 [01:07<00:40,  9.63it/s]
Loading 0:  62%|██████▏   | 614/995 [01:07<00:30, 12.31it/s]
Loading 0:  63%|██████▎   | 626/995 [01:07<00:17, 21.39it/s]
Loading 0:  64%|██████▍   | 635/995 [01:07<00:12, 28.88it/s]
Loading 0:  65%|██████▍   | 643/995 [01:08<00:15, 22.10it/s]
Loading 0:  65%|██████▌   | 649/995 [01:08<00:13, 25.50it/s]
Loading 0:  66%|██████▌   | 656/995 [01:08<00:11, 30.72it/s]
Loading 0:  67%|██████▋   | 662/995 [01:08<00:09, 33.37it/s]
Loading 0:  67%|██████▋   | 668/995 [01:08<00:09, 35.10it/s]
Loading 0:  68%|██████▊   | 673/995 [01:08<00:09, 35.59it/s]
Loading 0:  69%|██████▊   | 684/995 [01:08<00:06, 48.25it/s]
Loading 0:  69%|██████▉   | 691/995 [01:09<00:10, 29.62it/s]
Loading 0:  70%|██████▉   | 696/995 [01:09<00:09, 31.46it/s]
Loading 0:  70%|███████   | 701/995 [01:09<00:08, 33.89it/s]
Loading 0:  71%|███████   | 706/995 [01:09<00:07, 36.25it/s]
Loading 0:  71%|███████▏  | 711/995 [01:09<00:07, 39.08it/s]
Loading 0:  72%|███████▏  | 718/995 [01:09<00:06, 45.05it/s]
Loading 0:  73%|███████▎  | 724/995 [01:09<00:06, 44.29it/s]
Loading 0:  73%|███████▎  | 729/995 [01:10<00:06, 43.84it/s]
Loading 0:  74%|███████▍  | 735/995 [01:10<00:05, 47.82it/s]
Loading 0:  74%|███████▍  | 741/995 [01:10<00:09, 25.73it/s]
Loading 0:  75%|███████▌  | 748/995 [01:10<00:07, 31.44it/s]
Loading 0:  76%|███████▌  | 753/995 [01:10<00:07, 33.01it/s]
Loading 0:  76%|███████▌  | 758/995 [01:11<00:06, 34.58it/s]
Loading 0:  77%|███████▋  | 763/995 [01:11<00:06, 36.09it/s]
Loading 0:  77%|███████▋  | 768/995 [01:11<00:06, 36.77it/s]
Loading 0:  78%|███████▊  | 778/995 [01:11<00:04, 50.83it/s]
Loading 0:  79%|███████▉  | 784/995 [01:11<00:04, 46.63it/s]
Loading 0:  79%|███████▉  | 790/995 [01:11<00:04, 49.00it/s]
Loading 0:  80%|████████  | 796/995 [01:12<00:07, 25.91it/s]
Loading 0:  81%|████████  | 801/995 [01:12<00:06, 28.02it/s]
Loading 0:  81%|████████  | 805/995 [01:12<00:06, 29.07it/s]
Loading 0:  82%|████████▏ | 811/995 [01:12<00:05, 34.17it/s]
Loading 0:  82%|████████▏ | 816/995 [01:12<00:05, 34.40it/s]
Loading 0:  83%|████████▎ | 821/995 [01:12<00:05, 34.61it/s]
Loading 0:  83%|████████▎ | 825/995 [01:12<00:04, 34.62it/s]
Loading 0:  83%|████████▎ | 829/995 [01:13<00:04, 35.10it/s]
Loading 0:  84%|████████▍ | 840/995 [01:13<00:03, 51.63it/s]
Loading 0:  85%|████████▍ | 841/995 [01:25<00:02, 51.63it/s]
Loading 0:  85%|████████▍ | 842/995 [01:37<03:37,  1.42s/it]
Loading 0:  85%|████████▌ | 849/995 [01:37<02:15,  1.08it/s]
Loading 0:  86%|████████▌ | 854/995 [01:38<01:36,  1.47it/s]
Loading 0:  86%|████████▋ | 859/995 [01:38<01:07,  2.02it/s]
Loading 0:  87%|████████▋ | 864/995 [01:38<00:47,  2.78it/s]
Loading 0:  88%|████████▊ | 871/995 [01:38<00:29,  4.27it/s]
Loading 0:  88%|████████▊ | 877/995 [01:38<00:20,  5.89it/s]
Loading 0:  89%|████████▊ | 882/995 [01:38<00:14,  7.62it/s]
Loading 0:  89%|████████▉ | 887/995 [01:38<00:10,  9.86it/s]
Loading 0:  90%|████████▉ | 892/995 [01:38<00:08, 12.74it/s]
Loading 0:  90%|█████████ | 897/995 [01:39<00:07, 12.43it/s]
Loading 0:  91%|█████████ | 904/995 [01:39<00:05, 17.46it/s]
Loading 0:  91%|█████████▏| 909/995 [01:39<00:04, 20.43it/s]
Loading 0:  92%|█████████▏| 914/995 [01:39<00:03, 23.34it/s]
Loading 0:  92%|█████████▏| 919/995 [01:39<00:02, 26.56it/s]
Loading 0:  93%|█████████▎| 924/995 [01:39<00:02, 29.34it/s]
Loading 0:  94%|█████████▍| 935/995 [01:40<00:01, 42.95it/s]
Loading 0:  95%|█████████▍| 941/995 [01:40<00:01, 42.48it/s]
Loading 0:  95%|█████████▌| 947/995 [01:40<00:01, 44.37it/s]
Loading 0:  96%|█████████▌| 953/995 [01:42<00:04,  8.77it/s]
Loading 0:  96%|█████████▋| 958/995 [01:42<00:03, 10.97it/s]
Loading 0:  97%|█████████▋| 964/995 [01:42<00:02, 14.54it/s]
Loading 0:  97%|█████████▋| 969/995 [01:42<00:01, 17.52it/s]
Loading 0:  98%|█████████▊| 974/995 [01:42<00:01, 20.87it/s]
Loading 0:  98%|█████████▊| 979/995 [01:42<00:00, 24.33it/s]
Loading 0:  99%|█████████▉| 984/995 [01:43<00:00, 27.46it/s]
                                                            
/opt/conda/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py:919: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
mistralai-mixtral-8x7b-3473-v68-mkmlizer:   warnings.warn(
mistralai-mixtral-8x7b-3473-v68-mkmlizer: /opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
mistralai-mixtral-8x7b-3473-v68-mkmlizer:   warnings.warn(
mistralai-mixtral-8x7b-3473-v68-mkmlizer: /opt/conda/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py:769: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
mistralai-mixtral-8x7b-3473-v68-mkmlizer:   warnings.warn(
mistralai-mixtral-8x7b-3473-v68-mkmlizer: /opt/conda/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py:468: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.
mistralai-mixtral-8x7b-3473-v68-mkmlizer:   warnings.warn(
mistralai-mixtral-8x7b-3473-v68-mkmlizer: 
Downloading shards:   0%|          | 0/2 [00:00<?, ?it/s]
Downloading shards:  50%|█████     | 1/2 [00:07<00:07,  7.28s/it]
Downloading shards: 100%|██████████| 2/2 [00:09<00:00,  4.20s/it]
Downloading shards: 100%|██████████| 2/2 [00:09<00:00,  4.66s/it]
mistralai-mixtral-8x7b-3473-v68-mkmlizer: 
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
Loading checkpoint shards:  50%|█████     | 1/2 [00:00<00:00,  1.58it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00,  2.53it/s]
Loading checkpoint shards: 100%|██████████| 2/2 [00:00<00:00,  2.32it/s]
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Saving model to /tmp/reward_cache/reward.tensors
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Saving duration: 2.216s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Processed model ChaiML/gpt2_xl_pairwise_89m_step_347634 in 14.586s
mistralai-mixtral-8x7b-3473-v68-mkmlizer: creating bucket guanaco-reward-models
mistralai-mixtral-8x7b-3473-v68-mkmlizer: Bucket 's3://guanaco-reward-models/' created
mistralai-mixtral-8x7b-3473-v68-mkmlizer: uploading /tmp/reward_cache to s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/config.json s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/config.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/special_tokens_map.json s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/special_tokens_map.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/tokenizer_config.json s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/tokenizer_config.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/vocab.json s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/vocab.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/tokenizer.json s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/tokenizer.json
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/merges.txt s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/merges.txt
mistralai-mixtral-8x7b-3473-v68-mkmlizer: cp /tmp/reward_cache/reward.tensors s3://guanaco-reward-models/mistralai-mixtral-8x7b-3473-v68_reward/reward.tensors
Job mistralai-mixtral-8x7b-3473-v68-mkmlizer completed after 458.8s with status: succeeded
Stopping job with name mistralai-mixtral-8x7b-3473-v68-mkmlizer
Pipeline stage MKMLizer completed in 459.55s
Running pipeline stage MKMLKubeTemplater
Pipeline stage MKMLKubeTemplater completed in 0.10s
Running pipeline stage ISVCDeployer
Creating inference service mistralai-mixtral-8x7b-3473-v68
Waiting for inference service mistralai-mixtral-8x7b-3473-v68 to be ready
Connection pool is full, discarding connection: %s
Connection pool is full, discarding connection: %s
Connection pool is full, discarding connection: %s
Inference service mistralai-mixtral-8x7b-3473-v68 ready after 80.48571109771729s
Pipeline stage ISVCDeployer completed in 87.51s
Running pipeline stage StressChecker
Received healthy response to inference request in 3.1421103477478027s
Received healthy response to inference request in 1.7798562049865723s
Received healthy response to inference request in 1.8696303367614746s
Received healthy response to inference request in 2.045328378677368s
Received healthy response to inference request in 1.6569926738739014s
5 requests
0 failed requests
5th percentile: 1.6815653800964356
10th percentile: 1.7061380863189697
20th percentile: 1.755283498764038
30th percentile: 1.7978110313415527
40th percentile: 1.8337206840515137
50th percentile: 1.8696303367614746
60th percentile: 1.9399095535278321
70th percentile: 2.0101887702941896
80th percentile: 2.264684772491455
90th percentile: 2.703397560119629
95th percentile: 2.9227539539337157
99th percentile: 3.0982390689849852
mean time: 2.098783588409424
Pipeline stage StressChecker completed in 11.36s
mistralai-mixtral-8x7b-_3473_v68 status is now deployed due to DeploymentManager action
mistralai-mixtral-8x7b-_3473_v68 status is now inactive due to auto deactivation removed underperforming models

Start Time:

End Time:

Step (in hours or minutes, e.g. 1h, 10m):

Usage Metrics

Latency Metrics

Model Developer	Developer Rank	Model Rank	Model Rank By Developer	Elo Rating	Elo Rating By Developer	Auto Alignment Score	Auto Alignment Samples	User Alignment Score	User Alignment Question Responses	Submission ID	Parameters	Huggingface Repository	Variant	Number of Battles	Number of Battles By Developer	Win Ratio	Win Ratio By Developer	Type	Reward Repo	Best of	Max Input Tokens	Max Output Tokens	Model Architecture	Model Group	Date	Elo Norm	Alignment Norm	Alt Ranking	Alt Ranking By Developer	Elo Rank	Alignment Rank
chai_backend_admin	1	1	1	1273.34	1273.34		0	0.76	887.0	chaiml-elo-alignment-run-3_v25	8B	ChaiML/elo_alignment_run_3	e2e-test-chaiml-v25	10147	22459770	0.56	0.60	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-08-05	1.62	1.81	1	1	19	69
zonemercy	2	2	2	1275.12	1289.29		0	0.75	4615.0	zonemercy-burly-blue-cp2500_v5	8B	zonemercy/Burly_Blue_cp2500	zonemercy-burly-blue-cp2500_v5	7875	4840314	0.61	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	zonemercy/Burly_Blue_cp2	2024-07-19	1.65	1.71	2	2	12	79
PawanOsman	3	3	3	1277.56	1277.56		0	0.75	977.0	pawankrd-cosmosrp_v77	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v77	10992	4069431	0.58	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-20	1.71	1.43	3	3	6	144
zonemercy	2	4	2	1267.29	1289.29		0	0.75	1010.0	zonemercy-lexical-nemov9_2587_v1	13B	zonemercy/Lexical-Nemov9-1k5e6	zonemercy-lexical-nemov9_2587_v1	11494	4840314	0.55	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	8.0	1024.0	64.0	MistralForCausalLM	zonemercy/Lexical-Nemov9	2024-08-06	1.49	1.53	4	2	59	121
albert_chai	4	5	5	1272.80	1272.80		0	0.74	1338.0	chaiml-elo-alignment-run-3_v11	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	11278	18928	0.61	0.61	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-19	1.60	1.29	5	5	23	178
PawanOsman	3	6	3	1274.84	1277.56		0	0.74	1001.0	pawankrd-cosmosrp_v76	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v76	10908	4069431	0.58	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-20	1.65	1.15	6	3	13	215
PawanOsman	3	7	3	1273.13	1277.56		0	0.74	1068.0	pawankrd-cosmosrp_v79	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v79	10793	4069431	0.57	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-20	1.61	1.09	7	3	20	234
zonemercy	2	8	2	1274.77	1289.29		0	0.74	1017.0	chaiml-elo-alignment-run-3_v30	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v30	11476	4840314	0.56	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-08-06	1.65	0.95	11	2	14	299
PawanOsman	3	9	3	1274.05	1277.56		0	0.74	1197.0	pawankrd-cosmosrp_v78	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v78	12920	4069431	0.58	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-20	1.63	0.98	10	3	16	286
PawanOsman	3	10	3	1273.11	1277.56		0	0.73	1196.0	pawankrd-cosmosrp_v73	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v73	13819	4069431	0.61	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-18	1.61	0.93	12	3	21	309
PawanOsman	3	11	3	1268.48	1277.56		0	0.74	1548.0	pawankrd-cosmosrp_v72	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v72	12678	4069431	0.60	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-18	1.51	1.07	8	3	51	246
zonemercy	2	12	2	1273.79	1289.29		0	0.73	924.0	zonemercy-cogent-nemo-v1_4327_v1	13B	zonemercy/Cogent-Nemo-v1-1k1e5-ep1	zonemercy-cogent-nemo-v1_4327_v1	10226	4840314	0.56	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	8.0	1024.0	64.0	MistralForCausalLM	zonemercy/Cogent-Nemo-v1	2024-08-07	1.63	0.84	16	2	18	343
chai_backend_admin	1	13	1	1263.17	1273.34		0	0.74	866.0	chaiml-elo-alignment-run-3_v26	8B	ChaiML/elo_alignment_run_3	e2e-test-chaiml-v26	9925	22459770	0.55	0.60	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-08-05	1.40	1.19	9	1	102	197
zonemercy	2	14	2	1273.95	1289.29		0	0.73	10695.0	chaiml-elo-alignment-run-3_v8	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v8	210506	4840314	0.58	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-15	1.63	0.73	29	2	17	412
PawanOsman	3	15	3	1272.23	1277.56		0	0.73	1746.0	pawankrd-cosmosrp_v71	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v71	24610	4069431	0.62	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-17	1.59	0.79	22	3	25	376
PawanOsman	3	16	3	1269.64	1277.56		0	0.73	5002.0	pawankrd-cosmosrp_v69	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v69	132481	4069431	0.59	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-16	1.54	0.84	18	3	41	342
zonemercy	2	17	2	1270.04	1289.29		0	0.73	9211.0	chaiml-elo-alignment-run-3_v5	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	94158	4840314	0.55	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-12	1.55	0.81	21	2	37	363
zonemercy	2	18	2	1268.38	1289.29		0	0.73	5476.0	chaiml-elo-alignment-run-3_v4	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	50315	4840314	0.57	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-12	1.51	0.82	23	2	53	353
Jellywibble	5	19	19	1267.07	1276.15		0	0.73	5329.0	jellywibble-lora-120k-pr_2801_v2	8B	Jellywibble/lora_120k_pref_data_ep3_stacked_elo_alignment	nitral-ai-hathor-l3-8b-v-01_v1	30298	7536455	0.60	0.61	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	Jellywibble/lora_120k_pr	2024-07-09	1.48	0.86	19	19	62	333
zonemercy	2	20	2	1265.75	1289.29		0	0.73	5639.0	chaiml-elo-alignment-run-3_v3	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	11314	4840314	0.62	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-12	1.45	0.87	25	2	80	330
Jellywibble	5	21	19	1265.68	1276.15		0	0.73	1032.0	chaiml-elo-alignment-run-3_v18	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	11675	7536455	0.58	0.61	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-30	1.45	0.87	24	19	81	328
zonemercy	2	22	2	1264.17	1289.29		0	0.73	10173.0	zonemercy-burly-blue-cp2500_v1	8B	zonemercy/Burly_Blue_cp2500	zonemercy-burly-blue-cp2500_v1	124947	4840314	0.59	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	zonemercy/Burly_Blue_cp2	2024-07-14	1.42	0.87	27	2	96	329
PawanOsman	3	23	3	1267.20	1277.56		0	0.73	6358.0	pawankrd-cosmosrp_v65	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v65	64112	4069431	0.58	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-13	1.48	0.72	37	3	60	424
PawanOsman	3	24	3	1272.06	1277.56		0	0.72	50018.0	pawankrd-cosmosrp_v66	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v66	353310	4069431	0.56	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-13	1.59	0.51	66	3	28	574
Bbbrun0	6	25	25	1267.14	1267.14		0	0.73	1019.0	jellywibble-wibblewobble_v1	8B	Jellywibble/WibbleWobble	auto	11246	3305336	0.57	0.59	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	Jellywibble/WibbleWobble	2024-08-04	1.48	0.70	41	13	61	436
chai_backend_admin	1	26	1	1266.53	1273.34		0	0.73	862.0	chaiml-elo-alignment-run-3_v27	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	9878	22459770	0.57	0.60	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-08-05	1.47	0.72	39	1	74	420
zonemercy	2	27	2	1269.60	1289.29		0	0.73	5801.0	chaiml-elo-alignment-run-3_v6	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	52400	4840314	0.57	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-12	1.54	0.59	52	2	42	505
zonemercy	2	28	2	1264.08	1289.29		0	0.73	5613.0	jellywibble-lora-120k-pr_2801_v6	8B	Jellywibble/lora_120k_pref_data_ep3_stacked_elo_alignment	jellywibble-lora-120k-pr_2801_v6	11193	4840314	0.62	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	Jellywibble/lora_120k_pr	2024-07-12	1.42	0.80	33	2	98	368
PawanOsman	3	29	3	1266.54	1277.56		0	0.73	10925.0	pawankrd-cosmosrp_v64	8B	PawanKrd/CosmosRP	pawankrd-cosmosrp_v64	112573	4069431	0.56	0.75	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	PawanKrd/CosmosRP	2024-07-12	1.47	0.68	45	3	73	443
shiroe40	7	30	30	1275.61	1277.83		0	0.72	1010.0	cycy233-l3-ba-e-v-c2_v1	8B	cycy233/L3-ba-e-v-c2	auto	11705	11611290	0.57	0.61	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	cycy233/L3-ba-e-v-c2	2024-07-29	1.66	0.32	117	15	10	747
Bbbrun0	6	31	25	1245.66	1267.14		0	0.75	861.0	bbchicago-brt-8b-l3-v1-9_2176_v1	8B	BBChicago/Brt-8B-L3_v1.9_with_dpo_lora_s5900	bbchicago-brt-v1_9-dpo-s5900	19612	3305336	0.55	0.59	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	BBChicago/Brt-8B-L3_v1.9	2024-07-24	1.02	1.51	13	13	211	126
zonemercy	2	32	2	1264.76	1289.29		0	0.73	12325.0	chaiml-elo-alignment-run_5486_v1	8B	ChaiML/elo_alignment_run_3_ep1	dpo-v2-0710-rm-score-completed	28117	4840314	0.60	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-12	1.43	0.74	38	2	89	401
Jellywibble	5	33	19	1276.15	1276.15		0	0.72	954.0	chaiml-elo-alignment-run-3_v16	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	10540	7536455	0.60	0.61	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-30	1.68	0.29	130	19	9	770
zonemercy	2	34	2	1277.63	1289.29		0	0.72	12582.0	chaiml-elo-run-3-cp7000_v1	8B	ChaiML/elo_run_3_cp7000	nitral-ai-hathor-l3-8b-v-01_v1	24560	4840314	0.62	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_run_3_cp7000	2024-07-12	1.71	0.21	158	2	5	842
zonemercy	2	35	2	1246.89	1289.29		0	0.75	5478.0	chaiml-elo-alignment-run-3_v7	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	57438	4840314	0.58	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	4.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-13	1.05	1.44	14	2	202	140
shiroe40	7	36	30	1270.78	1277.83		0	0.72	1741.0	cycy233-l3-e-v2-c1_v2	8B	cycy233/L3-e-v2-c1	auto	6789	11611290	0.58	0.61	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	cycy233/L3-e-v2-c1	2024-07-19	1.56	0.48	77	15	33	599
zonemercy	2	37	2	1235.10	1289.29		0	0.76	919.0	zonemercy-lexical-nemov6_2011_v6	13B	zonemercy/Lexical-Nemov6-1k1e5	0805v1-1	9881	4840314	0.51	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	4.0	1024.0	64.0	MistralForCausalLM	zonemercy/Lexical-Nemov6	2024-08-05	0.79	1.89	26	2	351	59
Jellywibble	5	38	19	1266.70	1276.15	-0.91	34499	0.73	424998.0	chaiml-elo-alignment-run-3_v17	8B	ChaiML/elo_alignment_run_3	chaiml-elo-alignment-run-3_v3	5002486	7536455	0.57	0.61	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_alignment_run	2024-07-30	1.47	0.63	51	19	67	476
zonemercy	2	39	2	1283.66	1289.29		0	0.71	5498.0	chaiml-elo-run-3-cp7000_v4	8B	ChaiML/elo_run_3_cp7000	nitral-ai-hathor-l3-8b-v-01_v1	60274	4840314	0.64	0.64	basic	ChaiML/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	ChaiML/elo_run_3_cp7000	2024-07-13	1.84	-0.05	267	2	3	1088
shiroe40	7	40	30	1247.29	1277.83		0	0.75	1005.0	cycy233-l3-bc-v0-c1_v1	8B	cycy233/L3-bc-v0-c1	auto	11910	11611290	0.56	0.61	basic	Jellywibble/gpt2_xl_pairwise_89m_step_347634	16.0	512.0	64.0	LlamaForCausalLM	cycy233/L3-bc-v0-c1	2024-07-20	1.06	1.39	15	15	198	153