{
  "schema_version": 2,
  "model": "SlayerLab/pollock-mini-lm-125m",
  "release": "Pollock 1.4",
  "model_revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",
  "model_parameters": 127674624,
  "run_date": "2026-09-03",
  "protocol": {
    "harness": "lm-evaluation-harness 0.4.12",
    "transformers": "4.46.3",
    "torch": "2.4.1",
    "few_shot": 0,
    "limit": null,
    "batch_size": 64,
    "device": "NVIDIA GeForce RTX 3090 (CUDA)",
    "dtype": "bfloat16",
    "seed": 1337,
    "likelihood_requests": 261887,
    "evaluation_time_seconds": 341.51713780400314
  },
  "results": {
    "mmlu_continuation": {
      "metric": "acc",
      "value": 0.2551630821820253,
      "stderr": 0.003666351492479532,
      "samples": 14042
    },
    "arc_easy": {
      "metric": "acc_norm",
      "value": 0.4196127946127946,
      "stderr": 0.01012631584089153,
      "samples": 2376
    },
    "arc_challenge": {
      "metric": "acc_norm",
      "value": 0.2431740614334471,
      "stderr": 0.01253655414458709,
      "samples": 1172
    },
    "arc_mean": {
      "metric": "arithmetic_mean_of_acc_norm",
      "value": 0.3313934280231209,
      "samples": 3548
    },
    "piqa": {
      "metric": "acc_norm",
      "value": 0.6033732317736671,
      "stderr": 0.011413778810510453,
      "samples": 1838
    },
    "hellaswag": {
      "metric": "acc_norm",
      "value": 0.2997410874327823,
      "stderr": 0.0045720816569656715,
      "samples": 10042
    },
    "openbookqa": {
      "metric": "acc_norm",
      "value": 0.278,
      "stderr": 0.02005583388807091,
      "samples": 500
    },
    "winogrande": {
      "metric": "acc",
      "value": 0.49171270718232046,
      "stderr": 0.014050555322824194,
      "samples": 1267
    },
    "five_task_mean": {
      "metric": "unweighted_mean_of_arc_mean_piqa_hellaswag_openbookqa_winogrande",
      "value": 0.40084409088237816
    },
    "blimp": {
      "metric": "acc",
      "value": 0.7807313432835822,
      "stderr": 0.0014203717333328017,
      "samples": 67000
    },
    "sciq": {
      "metric": "acc_norm",
      "value": 0.656,
      "stderr": 0.015029633724408947,
      "samples": 1000
    },
    "lambada_openai": {
      "accuracy": 0.2782844944692412,
      "accuracy_stderr": 0.00624366502869379,
      "perplexity": 53.53963424828631,
      "perplexity_stderr": 2.011893913974489,
      "samples": 5153
    },
    "wikitext_2": {
      "bits_per_byte": 1.0071358793018934,
      "byte_perplexity": 2.0099169346632757,
      "word_perplexity": 41.80516816662532,
      "documents": 62
    }
  },
  "raw_aggregate_sha256": "df00efbe1f26635cd6e8386d7b175c2cf0f2189f92b2a7bdedf95a30aa4a78d0"
}