{
  "slug": "lmms-eval",
  "name": "LMMs-Eval",
  "category": "evals",
  "kind": "repo",
  "tier": "catalog",
  "what": "Evaluation toolkit for multimodal models.",
  "what_source": "editor",
  "why": "Measures how close you are to the frontier.",
  "icon": "Gauge",
  "stack_layer": "tools",
  "repo": "https://github.com/EvolvingLMMs-Lab/lmms-eval",
  "homepage": "https://www.lmms-lab.com",
  "readme": "https://github.com/EvolvingLMMs-Lab/lmms-eval#readme",
  "license": "Other",
  "repo_meta": {
    "stars": 4453,
    "last_push": "2026-10-15",
    "last_commit": "2026-10-15",
    "archived": false,
    "fork": false,
    "language": "Python",
    "checked_on": "2026-10-15"
  },
  "tags": [
    "license-unclear"
  ],
  "seen_in_videos": [],
  "cloud_note": null,
  "install": [
    {
      "run": "git clone https://github.com/EvolvingLMMs-Lab/lmms-eval.git",
      "platform": "any",
      "source": "README (auto-extracted)",
      "status": "unverified, not executed"
    },
    {
      "run": "uv pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git",
      "platform": "any",
      "source": "README (auto-extracted)",
      "status": "unverified, not executed"
    },
    {
      "run": "conda install openjdk=8",
      "platform": "any",
      "source": "README (auto-extracted)",
      "status": "unverified, not executed"
    },
    {
      "run": "pip install s2wrapper@git+https://github.com/bfshi/scaling_on_scales",
      "platform": "any",
      "source": "README (auto-extracted)",
      "status": "unverified, not executed"
    }
  ],
  "verify": [],
  "rollback": [
    {
      "run": "rm -rf lmms-eval",
      "source": "derived inverse of install command",
      "status": "unverified"
    },
    {
      "run": "pip uninstall -y s2wrapper@git+https://github.com/bfshi/scaling_on_scales",
      "source": "derived inverse of install command",
      "status": "unverified"
    }
  ],
  "gap_score": {
    "value": null,
    "status": "pending test on Mac Studio"
  },
  "verified_on": "2026-10-15"
}