{ "schema_version": 1, "date": "2026-09-30", "project": "Fusion Playground", "author": "Jan Tyl", "scope": "Dataset selection and configuration notes for the final Exp21/Exp22 comparison. Not a complete executable reproduction package.", "limitations": [ "Dataset and model commit revisions were not recovered for this manifest. Source links may change.", "Task IDs are experiment-local identifiers, sometimes row indices into the frozen inputs, not always upstream stable IDs.", "The AIME links are sources named in the preparation script; it also had fallback repositories. The exact successful source revision still needs to be documented.", "Questions, answer keys and model outputs are not redistributed here. Obtain source datasets under their original access terms.", "The main set contains 2544 unique tasks; the strict router comparison uses 2537 due to seven IFEval items missing required outputs.", "Later exploratory analyses reused the held-out A8 set.", "Observed configuration examples do not exhaust rerun limits; consult the future complete run manifest for exact per-output settings." ], "datasets": { "gpqa": { "source_urls": [ "https://huggingface.co/datasets/Idavidrein/gpqa" ], "count": 198, "task_ids": [ "gpqa:rec0Arme2jcXQZnAW", "gpqa:rec2UlKqC6RFHdcro", "gpqa:rec2ZTsqazBZ3LTWZ", "gpqa:rec527dNeEtWJrYNl", "gpqa:rec5eA17Qr1ucNLUf", "gpqa:rec5rjeLsEq5Fg7Oj", "gpqa:rec7C06XOB2h298ue", "gpqa:rec8y3ZrBOcLgNEkE", "gpqa:recBXlqrZUCR897It", "gpqa:recBtVK8rBVtIlXDq", "gpqa:recDDxpS9s8cwkqfq", "gpqa:recDytVnNYZe2HuUU", "gpqa:recG1ljJQcVy6CbSl", "gpqa:recGFKDUd8qgz81Zl", "gpqa:recI1ls9OXdxatHQn", "gpqa:recIDiDKKrN61Auyr", "gpqa:recIIUynGGpsGEYuo", "gpqa:recINSFEYLCyyd08m", "gpqa:recL9MFV5zmdlle5T", "gpqa:recLSzwBUS3olCjhl", "gpqa:recNFJjE5PPTqVJGv", "gpqa:recNu3MXkvWUzHZr9", "gpqa:recNuT2oSnO86bxOx", "gpqa:recQ8oxSY2aCwFRS8", "gpqa:recUBgVlkKzcRPDdK", "gpqa:recUOePh79cp4T2Bg", "gpqa:recV1nqYQvpII94oC", "gpqa:recVvpD8miVjmmyfe", "gpqa:recZSGUkn56v9kEp1", "gpqa:recdya6FuYraBU5Rh", "gpqa:receo3UUzRvWCqyfA", "gpqa:rechKl68Uc6H7vU0N", "gpqa:reciuppdQde4FGX1z", "gpqa:recjJ54TXc04enRkZ", "gpqa:reckYqR92oJkqJXkE", "gpqa:recmI7EiLv72PxmYK", "gpqa:recmkvk6EFAmqyMxR", "gpqa:recnGEpF1srQpaqWq", "gpqa:recnjViFrqlZNL3fY", "gpqa:recpJmTOXx6T48nDJ", "gpqa:recr3VHM4zYf6dMFY", "gpqa:recrHBEJJoDTV05JR", "gpqa:rectXfsCM1dj4Kv2c", "gpqa:recuYEkVjdVUwA3c0", "gpqa:recuyeuT5rQ6qDt8F", "gpqa:reczjcMtrB1YGS2fO", "gpqa:rec1oj2DveQWl9Rpw", "gpqa:recAYkd96NNuNl1Ei", "gpqa:recE2ihVfqEK4R9d0", "gpqa:recGee5m84dg5FZkc", "gpqa:recLB0EkQ54bYVhnd", "gpqa:recMicVBcqy1xM1jq", "gpqa:recPIzpnuYpB4yvmp", "gpqa:recWXwn9v4IG9ZrM6", "gpqa:recXsYa3i2UhGF5fe", "gpqa:recYCMJWH7IpcdqwE", "gpqa:recatj4YXzbOaq3Ar", "gpqa:recixxJmdux0d8LZQ", "gpqa:reckEnrOPFT9Ru7tW", "gpqa:recwW1A85nfyQpReG", "gpqa:rec06pnAkLOr2t2mp", "gpqa:rec0wZvZgiz320KRs", "gpqa:rec0yTRmO1o1xCA6H", "gpqa:rec1zl5LvaatzGhFt", "gpqa:rec260hNUCEj109Dj", "gpqa:rec2fsnzUuvNtUYK8", "gpqa:rec4JrWy7Il8ho5vk", "gpqa:rec4L69T0Y1AS4AFS", "gpqa:rec4UqStf9WUVif1f", "gpqa:rec4r4KWNLrT7g2kR", "gpqa:rec7qmSnbud4FHSqL", "gpqa:rec8nshandHARTkrg", "gpqa:rec9GugrngvsQR99A", "gpqa:rec9ubQihAh6g9bft", "gpqa:recA1i5ZAh0Uzclxp", "gpqa:recAAJoHMW45Lv5je", "gpqa:recBGxVySx1pkrO1i", "gpqa:recCJJOeBGERaHYax", "gpqa:recD8oX1KevFbl7bL", "gpqa:recDj2Y2BbtV02Wv5", "gpqa:recEmTBhx2hgw6tPQ", "gpqa:recFaL6j8UMhutXrc", "gpqa:recGFNRVl1qBZGwyU", "gpqa:recHVUd2DkoKfedJZ", "gpqa:recIMcQfaQNa6vzsT", "gpqa:recIOlKBsOeEcgkA1", "gpqa:recIj8lR4tuDgrHou", "gpqa:recIlghoOw5VbY4Zh", "gpqa:recJJcOHhFeSaRpl4", "gpqa:recJZ3QEfRKjYw9a7", "gpqa:recK9F5aqdaybl8bb", "gpqa:recKcjTwONILZqHAX", "gpqa:recKm6LNWykGapmCr", "gpqa:recKv4hFyNpViUXnL", "gpqa:recLAgwx9vbgB5EHk", "gpqa:recLrRirj7iXoB0rU", "gpqa:recN4DY9Q5V03glmQ", "gpqa:recNAiPUKOulQbKVP", "gpqa:recO3hvCWRGiG0odN", "gpqa:recOYsaYs6RmtlTDy", "gpqa:recOkA2zsy4GSMD9R", "gpqa:recOvqPKUtyy9ISA1", "gpqa:recPL4ZPpVYxgMu57", "gpqa:recPSTGXK3P39yNYT", "gpqa:recPzW1WqRnPs57D6", "gpqa:recQiWOxhkXRz9NVE", "gpqa:recRTiyaRymerDJ4y", "gpqa:recRgabRzMaEoBRcM", "gpqa:recS48OsU6kVadBPW", "gpqa:recSBcGLPatKb3Ygu", "gpqa:recTs7qzfJs6kfLUK", "gpqa:recVEXvhYxAhELVqB", "gpqa:recWPMm91mGDLXONm", "gpqa:recWxGU8Q4YReJ1tb", "gpqa:recWzUUYBN0NuXO2k", "gpqa:recXsuOHRBLcyenF2", "gpqa:recXvQ6gWAmyakrpD", "gpqa:recYA3LPsCvF1fTMI", "gpqa:recYOzCsevNz61Lyn", "gpqa:recYt8xx80OTyDsL0", "gpqa:recZ13cwgDQf9jRd9", "gpqa:recZWeueB7lSPR6wN", "gpqa:recZbxrocrxh9YENH", "gpqa:recZdZxsN0AxDx0pB", "gpqa:recZt4x514UQNnsgy", "gpqa:reca44yABeO2fx7UB", "gpqa:recaXdGn3FAIkxLGM", "gpqa:recafnH3RRnMBXEb1", "gpqa:recakFKZioXUuldcd", "gpqa:recb2M22zaD3tL6Qc", "gpqa:recb4cGsC6BJUCU3V", "gpqa:reccOKzFNmyqeJ6ry", "gpqa:reccVFW11Elovnf8T", "gpqa:recclFbsjbaiVVnnV", "gpqa:receYQxnUBvxPcvbQ", "gpqa:recf6ayQmL1SxKbvW", "gpqa:recgI6tUQ7RLJRWGx", "gpqa:recgM7o1tcc7tP778", "gpqa:recgXxEgllSGEpELP", "gpqa:rechgQucGlrnt8KRV", "gpqa:rechtUKxC6G67H7Pr", "gpqa:recihePFulRgNKsIn", "gpqa:recjgMJaMxz4ESDF2", "gpqa:reck1zKeCssmN82Ga", "gpqa:reck4G4xxv3YnpbtQ", "gpqa:reckl6PkaEMWM50yZ", "gpqa:recl1UtgTVKishAq4", "gpqa:recmwwQJnx7ll7bqL", "gpqa:recn3NhOhqAPLda16", "gpqa:recnTTKdBzfuoZ7w7", "gpqa:recnawZPumSCLMVgp", "gpqa:reco8WSc71p8zAv4q", "gpqa:recooG6bivTUJpDBz", "gpqa:recoy9ZLBsc7HIRBy", "gpqa:recpki12iG9RUGrz9", "gpqa:recqFDvYKKRmbnoUb", "gpqa:recqGD3fxPCI59vPQ", "gpqa:recqLRu4BcJvlSANc", "gpqa:recrNbtgTNoabJJi6", "gpqa:recrs0kUo7Ua1xhKO", "gpqa:recs3PLPUEMiqg4P8", "gpqa:rect4iLrSfUwkNTno", "gpqa:rectlyG9pCAAuWhoB", "gpqa:recv7GsQg3f0fvB1f", "gpqa:recvZlMbpPz0yLDle", "gpqa:recvlnu8BuLmXZhSX", "gpqa:recw4rROcnHKNZhtK", "gpqa:recxTy0x7M6T9ikc0", "gpqa:recxsX66ktJ1Lwt4f", "gpqa:recyPffUDqC2k7ZZO", "gpqa:recyl3usDqb7ruXJx", "gpqa:recypVp2NmPlBKVTp", "gpqa:reczQ4I0VpENdMtIj", "gpqa:reczUoM8JsxU6pYxr", "gpqa:reczzzihL7btBH7RO", "gpqa:rec0VuKUjt1SZ7NYv", "gpqa:rec32C1ZEapBnCC0E", "gpqa:rec47oNPXudlTIk4y", "gpqa:rec6sE2CRtD4drtHg", "gpqa:rec9W28HgpEUeUN8k", "gpqa:recBhnXrUyTJ6WHIR", "gpqa:recINGR1z01Fh1Z3A", "gpqa:recJpyGtGIsxulevT", "gpqa:recRZjaElf8ft7xTq", "gpqa:recReg13iV2HwJTaA", "gpqa:recRhwQwHijW0gCLo", "gpqa:recUc29lMDBEvurYo", "gpqa:recVE8cUNHpHZIAvL", "gpqa:recb80OwMgNnceA9t", "gpqa:reccVBrYdwsB84fGy", "gpqa:recfTlTMjZBuOducT", "gpqa:recgC4GJ8wUemlUQD", "gpqa:recoiTJPGUmzAkief", "gpqa:recwlwlcPvXVNPmvl", "gpqa:recyrk6XBVwugG5u8", "gpqa:recywRj5a8EEjj2Ib", "gpqa:recz3MfYHPRhsEQkW", "gpqa:reczkBiPPNrNN49Hp" ] }, "math500": { "source_urls": [ "https://huggingface.co/datasets/HuggingFaceH4/MATH-500" ], "count": 500, "task_ids": [ "math500:math500-0000", "math500:math500-0001", "math500:math500-0002", "math500:math500-0003", "math500:math500-0004", "math500:math500-0005", "math500:math500-0006", "math500:math500-0007", "math500:math500-0008", "math500:math500-0009", "math500:math500-0010", "math500:math500-0011", "math500:math500-0012", "math500:math500-0013", "math500:math500-0014", "math500:math500-0015", "math500:math500-0016", "math500:math500-0017", "math500:math500-0018", "math500:math500-0019", "math500:math500-0020", "math500:math500-0021", "math500:math500-0022", "math500:math500-0023", "math500:math500-0024", "math500:math500-0025", "math500:math500-0026", "math500:math500-0027", "math500:math500-0028", "math500:math500-0029", "math500:math500-0030", "math500:math500-0031", "math500:math500-0032", "math500:math500-0033", "math500:math500-0034", "math500:math500-0035", "math500:math500-0036", "math500:math500-0037", "math500:math500-0038", "math500:math500-0039", "math500:math500-0040", "math500:math500-0041", "math500:math500-0042", "math500:math500-0043", "math500:math500-0044", "math500:math500-0045", "math500:math500-0046", "math500:math500-0047", "math500:math500-0048", "math500:math500-0049", "math500:math500-0050", "math500:math500-0051", "math500:math500-0052", "math500:math500-0053", "math500:math500-0054", "math500:math500-0055", "math500:math500-0056", "math500:math500-0057", "math500:math500-0058", "math500:math500-0059", "math500:math500-0060", "math500:math500-0061", "math500:math500-0062", "math500:math500-0063", "math500:math500-0064", "math500:math500-0065", "math500:math500-0066", "math500:math500-0067", "math500:math500-0068", "math500:math500-0069", "math500:math500-0070", "math500:math500-0071", "math500:math500-0072", "math500:math500-0073", "math500:math500-0074", "math500:math500-0075", "math500:math500-0076", "math500:math500-0077", "math500:math500-0078", "math500:math500-0079", "math500:math500-0080", "math500:math500-0081", "math500:math500-0082", "math500:math500-0083", "math500:math500-0084", "math500:math500-0085", "math500:math500-0086", "math500:math500-0087", "math500:math500-0088", "math500:math500-0089", "math500:math500-0090", "math500:math500-0091", "math500:math500-0092", "math500:math500-0093", "math500:math500-0094", "math500:math500-0095", "math500:math500-0096", "math500:math500-0097", "math500:math500-0098", "math500:math500-0099", "math500:math500-0100", "math500:math500-0101", "math500:math500-0102", "math500:math500-0103", "math500:math500-0104", "math500:math500-0105", "math500:math500-0106", "math500:math500-0107", "math500:math500-0108", "math500:math500-0109", "math500:math500-0110", "math500:math500-0111", "math500:math500-0112", "math500:math500-0113", "math500:math500-0114", "math500:math500-0115", "math500:math500-0116", "math500:math500-0117", "math500:math500-0118", "math500:math500-0119", "math500:math500-0120", "math500:math500-0121", "math500:math500-0122", "math500:math500-0123", "math500:math500-0124", "math500:math500-0125", "math500:math500-0126", "math500:math500-0127", "math500:math500-0128", "math500:math500-0129", "math500:math500-0130", "math500:math500-0131", "math500:math500-0132", "math500:math500-0133", "math500:math500-0134", "math500:math500-0135", "math500:math500-0136", "math500:math500-0137", "math500:math500-0138", "math500:math500-0139", "math500:math500-0140", "math500:math500-0141", "math500:math500-0142", "math500:math500-0143", "math500:math500-0144", "math500:math500-0145", "math500:math500-0146", "math500:math500-0147", "math500:math500-0148", "math500:math500-0149", "math500:math500-0150", "math500:math500-0151", "math500:math500-0152", "math500:math500-0153", "math500:math500-0154", "math500:math500-0155", "math500:math500-0156", "math500:math500-0157", "math500:math500-0158", "math500:math500-0159", "math500:math500-0160", "math500:math500-0161", "math500:math500-0162", "math500:math500-0163", "math500:math500-0164", "math500:math500-0165", "math500:math500-0166", "math500:math500-0167", "math500:math500-0168", "math500:math500-0169", "math500:math500-0170", "math500:math500-0171", "math500:math500-0172", "math500:math500-0173", "math500:math500-0174", "math500:math500-0175", "math500:math500-0176", "math500:math500-0177", "math500:math500-0178", "math500:math500-0179", "math500:math500-0180", "math500:math500-0181", "math500:math500-0182", "math500:math500-0183", "math500:math500-0184", "math500:math500-0185", "math500:math500-0186", "math500:math500-0187", "math500:math500-0188", "math500:math500-0189", "math500:math500-0190", "math500:math500-0191", "math500:math500-0192", "math500:math500-0193", "math500:math500-0194", "math500:math500-0195", "math500:math500-0196", "math500:math500-0197", "math500:math500-0198", "math500:math500-0199", "math500:math500-0200", "math500:math500-0201", "math500:math500-0202", "math500:math500-0203", "math500:math500-0204", "math500:math500-0205", "math500:math500-0206", "math500:math500-0207", "math500:math500-0208", "math500:math500-0209", "math500:math500-0210", "math500:math500-0211", "math500:math500-0212", "math500:math500-0213", "math500:math500-0214", "math500:math500-0215", "math500:math500-0216", "math500:math500-0217", "math500:math500-0218", "math500:math500-0219", "math500:math500-0220", "math500:math500-0221", "math500:math500-0222", "math500:math500-0223", "math500:math500-0224", "math500:math500-0225", "math500:math500-0226", "math500:math500-0227", "math500:math500-0228", "math500:math500-0229", "math500:math500-0230", "math500:math500-0231", "math500:math500-0232", "math500:math500-0233", "math500:math500-0234", "math500:math500-0235", "math500:math500-0236", "math500:math500-0237", "math500:math500-0238", "math500:math500-0239", "math500:math500-0240", "math500:math500-0241", "math500:math500-0242", "math500:math500-0243", "math500:math500-0244", "math500:math500-0245", "math500:math500-0246", "math500:math500-0247", "math500:math500-0248", "math500:math500-0249", "math500:math500-0250", "math500:math500-0251", "math500:math500-0252", "math500:math500-0253", "math500:math500-0254", "math500:math500-0255", "math500:math500-0256", "math500:math500-0257", "math500:math500-0258", "math500:math500-0259", "math500:math500-0260", "math500:math500-0261", "math500:math500-0262", "math500:math500-0263", "math500:math500-0264", "math500:math500-0265", "math500:math500-0266", "math500:math500-0267", "math500:math500-0268", "math500:math500-0269", "math500:math500-0270", "math500:math500-0271", "math500:math500-0272", "math500:math500-0273", "math500:math500-0274", "math500:math500-0275", "math500:math500-0276", "math500:math500-0277", "math500:math500-0278", "math500:math500-0279", "math500:math500-0280", "math500:math500-0281", "math500:math500-0282", "math500:math500-0283", "math500:math500-0284", "math500:math500-0285", "math500:math500-0286", "math500:math500-0287", "math500:math500-0288", "math500:math500-0289", "math500:math500-0290", "math500:math500-0291", "math500:math500-0292", "math500:math500-0293", "math500:math500-0294", "math500:math500-0295", "math500:math500-0296", "math500:math500-0297", "math500:math500-0298", "math500:math500-0299", "math500:math500-0300", "math500:math500-0301", "math500:math500-0302", "math500:math500-0303", "math500:math500-0304", "math500:math500-0305", "math500:math500-0306", "math500:math500-0307", "math500:math500-0308", "math500:math500-0309", "math500:math500-0310", "math500:math500-0311", "math500:math500-0312", "math500:math500-0313", "math500:math500-0314", "math500:math500-0315", "math500:math500-0316", "math500:math500-0317", "math500:math500-0318", "math500:math500-0319", "math500:math500-0320", "math500:math500-0321", "math500:math500-0322", "math500:math500-0323", "math500:math500-0324", "math500:math500-0325", "math500:math500-0326", "math500:math500-0327", "math500:math500-0328", "math500:math500-0329", "math500:math500-0330", "math500:math500-0331", "math500:math500-0332", "math500:math500-0333", "math500:math500-0334", "math500:math500-0335", "math500:math500-0336", "math500:math500-0337", "math500:math500-0338", "math500:math500-0339", "math500:math500-0340", "math500:math500-0341", "math500:math500-0342", "math500:math500-0343", "math500:math500-0344", "math500:math500-0345", "math500:math500-0346", "math500:math500-0347", "math500:math500-0348", "math500:math500-0349", "math500:math500-0350", "math500:math500-0351", "math500:math500-0352", "math500:math500-0353", "math500:math500-0354", "math500:math500-0355", "math500:math500-0356", "math500:math500-0357", "math500:math500-0358", "math500:math500-0359", "math500:math500-0360", "math500:math500-0361", "math500:math500-0362", "math500:math500-0363", "math500:math500-0364", "math500:math500-0365", "math500:math500-0366", "math500:math500-0367", "math500:math500-0368", "math500:math500-0369", "math500:math500-0370", "math500:math500-0371", "math500:math500-0372", "math500:math500-0373", "math500:math500-0374", "math500:math500-0375", "math500:math500-0376", "math500:math500-0377", "math500:math500-0378", "math500:math500-0379", "math500:math500-0380", "math500:math500-0381", "math500:math500-0382", "math500:math500-0383", "math500:math500-0384", "math500:math500-0385", "math500:math500-0386", "math500:math500-0387", "math500:math500-0388", "math500:math500-0389", "math500:math500-0390", "math500:math500-0391", "math500:math500-0392", "math500:math500-0393", "math500:math500-0394", "math500:math500-0395", "math500:math500-0396", "math500:math500-0397", "math500:math500-0398", "math500:math500-0399", "math500:math500-0400", "math500:math500-0401", "math500:math500-0402", "math500:math500-0403", "math500:math500-0404", "math500:math500-0405", "math500:math500-0406", "math500:math500-0407", "math500:math500-0408", "math500:math500-0409", "math500:math500-0410", "math500:math500-0411", "math500:math500-0412", "math500:math500-0413", "math500:math500-0414", "math500:math500-0415", "math500:math500-0416", "math500:math500-0417", "math500:math500-0418", "math500:math500-0419", "math500:math500-0420", "math500:math500-0421", "math500:math500-0422", "math500:math500-0423", "math500:math500-0424", "math500:math500-0425", "math500:math500-0426", "math500:math500-0427", "math500:math500-0428", "math500:math500-0429", "math500:math500-0430", "math500:math500-0431", "math500:math500-0432", "math500:math500-0433", "math500:math500-0434", "math500:math500-0435", "math500:math500-0436", "math500:math500-0437", "math500:math500-0438", "math500:math500-0439", "math500:math500-0440", "math500:math500-0441", "math500:math500-0442", "math500:math500-0443", "math500:math500-0444", "math500:math500-0445", "math500:math500-0446", "math500:math500-0447", "math500:math500-0448", "math500:math500-0449", "math500:math500-0450", "math500:math500-0451", "math500:math500-0452", "math500:math500-0453", "math500:math500-0454", "math500:math500-0455", "math500:math500-0456", "math500:math500-0457", "math500:math500-0458", "math500:math500-0459", "math500:math500-0460", "math500:math500-0461", "math500:math500-0462", "math500:math500-0463", "math500:math500-0464", "math500:math500-0465", "math500:math500-0466", "math500:math500-0467", "math500:math500-0468", "math500:math500-0469", "math500:math500-0470", "math500:math500-0471", "math500:math500-0472", "math500:math500-0473", "math500:math500-0474", "math500:math500-0475", "math500:math500-0476", "math500:math500-0477", "math500:math500-0478", "math500:math500-0479", "math500:math500-0480", "math500:math500-0481", "math500:math500-0482", "math500:math500-0483", "math500:math500-0484", "math500:math500-0485", "math500:math500-0486", "math500:math500-0487", "math500:math500-0488", "math500:math500-0489", "math500:math500-0490", "math500:math500-0491", "math500:math500-0492", "math500:math500-0493", "math500:math500-0494", "math500:math500-0495", "math500:math500-0496", "math500:math500-0497", "math500:math500-0498", "math500:math500-0499" ] }, "aime": { "source_urls": [ "https://huggingface.co/datasets/HuggingFaceH4/aime_2024", "https://huggingface.co/datasets/yentinglin/aime_2025" ], "count": 60, "task_ids": [ "aime24:aime24-0000", "aime24:aime24-0001", "aime24:aime24-0002", "aime24:aime24-0003", "aime24:aime24-0004", "aime24:aime24-0005", "aime24:aime24-0006", "aime24:aime24-0007", "aime24:aime24-0008", "aime24:aime24-0009", "aime24:aime24-0010", "aime24:aime24-0011", "aime24:aime24-0012", "aime24:aime24-0013", "aime24:aime24-0014", "aime24:aime24-0015", "aime24:aime24-0016", "aime24:aime24-0017", "aime24:aime24-0018", "aime24:aime24-0019", "aime24:aime24-0020", "aime24:aime24-0021", "aime24:aime24-0022", "aime24:aime24-0023", "aime24:aime24-0024", "aime24:aime24-0025", "aime24:aime24-0026", "aime24:aime24-0027", "aime24:aime24-0028", "aime24:aime24-0029", "aime25:aime25-0000", "aime25:aime25-0001", "aime25:aime25-0002", "aime25:aime25-0003", "aime25:aime25-0004", "aime25:aime25-0005", "aime25:aime25-0006", "aime25:aime25-0007", "aime25:aime25-0008", "aime25:aime25-0009", "aime25:aime25-0010", "aime25:aime25-0011", "aime25:aime25-0012", "aime25:aime25-0013", "aime25:aime25-0014", "aime25:aime25-0015", "aime25:aime25-0016", "aime25:aime25-0017", "aime25:aime25-0018", "aime25:aime25-0019", "aime25:aime25-0020", "aime25:aime25-0021", "aime25:aime25-0022", "aime25:aime25-0023", "aime25:aime25-0024", "aime25:aime25-0025", "aime25:aime25-0026", "aime25:aime25-0027", "aime25:aime25-0028", "aime25:aime25-0029" ] }, "ifeval": { "source_urls": [ "https://huggingface.co/datasets/google/IFEval" ], "count": 541, "task_ids": [ "ifeval:1000", "ifeval:1001", "ifeval:1005", "ifeval:1012", "ifeval:1019", "ifeval:102", "ifeval:1021", "ifeval:1040", "ifeval:1051", "ifeval:1069", "ifeval:1072", "ifeval:1075", "ifeval:1082", "ifeval:1087", "ifeval:1092", "ifeval:1094", "ifeval:1098", "ifeval:1107", "ifeval:1108", "ifeval:1122", "ifeval:1127", "ifeval:1128", "ifeval:1129", "ifeval:1130", "ifeval:1131", "ifeval:1132", "ifeval:1137", "ifeval:1139", "ifeval:1147", "ifeval:1148", "ifeval:1153", "ifeval:1154", "ifeval:1162", "ifeval:1174", "ifeval:1180", "ifeval:1187", "ifeval:1203", "ifeval:1216", "ifeval:1217", "ifeval:1219", "ifeval:122", "ifeval:1220", "ifeval:1233", "ifeval:1236", "ifeval:1237", "ifeval:1242", "ifeval:1246", "ifeval:1248", "ifeval:1251", "ifeval:1258", "ifeval:1259", "ifeval:1262", "ifeval:1265", "ifeval:1268", "ifeval:127", "ifeval:1281", "ifeval:1286", "ifeval:1287", "ifeval:13", "ifeval:1300", "ifeval:1305", "ifeval:1307", "ifeval:1314", "ifeval:1322", "ifeval:1325", "ifeval:1332", "ifeval:1342", "ifeval:1348", "ifeval:136", "ifeval:1361", "ifeval:1367", "ifeval:1372", "ifeval:1375", "ifeval:1377", "ifeval:1379", "ifeval:1381", "ifeval:1389", "ifeval:1392", "ifeval:1393", "ifeval:1402", "ifeval:1418", "ifeval:142", "ifeval:143", "ifeval:1436", "ifeval:1446", "ifeval:1466", "ifeval:1476", "ifeval:1477", "ifeval:1480", "ifeval:1481", "ifeval:1498", "ifeval:1508", "ifeval:1512", "ifeval:1516", "ifeval:1518", "ifeval:152", "ifeval:1531", "ifeval:1535", "ifeval:1537", "ifeval:1546", "ifeval:1548", "ifeval:1551", "ifeval:1561", "ifeval:1566", "ifeval:1571", "ifeval:1580", "ifeval:1582", "ifeval:1591", "ifeval:1592", "ifeval:1593", "ifeval:16", "ifeval:1601", "ifeval:1609", "ifeval:1619", "ifeval:1620", "ifeval:1627", "ifeval:1629", "ifeval:163", "ifeval:1634", "ifeval:164", "ifeval:1643", "ifeval:1644", "ifeval:1645", "ifeval:1646", "ifeval:1653", "ifeval:1656", "ifeval:1658", "ifeval:1659", "ifeval:1667", "ifeval:167", "ifeval:1670", "ifeval:1675", "ifeval:168", "ifeval:1686", "ifeval:1691", "ifeval:1705", "ifeval:1713", "ifeval:1730", "ifeval:1733", "ifeval:1738", "ifeval:1739", "ifeval:1746", "ifeval:1759", "ifeval:1773", "ifeval:1776", "ifeval:1779", "ifeval:1781", "ifeval:179", "ifeval:1793", "ifeval:1800", "ifeval:1802", "ifeval:181", "ifeval:1813", "ifeval:1823", "ifeval:1825", "ifeval:1834", "ifeval:1837", "ifeval:1843", "ifeval:1845", "ifeval:1857", "ifeval:1858", "ifeval:1879", "ifeval:1880", "ifeval:1883", "ifeval:1886", "ifeval:1893", "ifeval:1897", "ifeval:19", "ifeval:1902", "ifeval:1906", "ifeval:1908", "ifeval:1922", "ifeval:1927", "ifeval:1928", "ifeval:1934", "ifeval:1936", "ifeval:1939", "ifeval:1943", "ifeval:1944", "ifeval:1954", "ifeval:1964", "ifeval:1967", "ifeval:1980", "ifeval:1996", "ifeval:1999", "ifeval:201", "ifeval:2010", "ifeval:2015", "ifeval:202", "ifeval:2023", "ifeval:2028", "ifeval:2034", "ifeval:2035", "ifeval:2041", "ifeval:2063", "ifeval:2069", "ifeval:2070", "ifeval:2071", "ifeval:2078", "ifeval:2084", "ifeval:209", "ifeval:2097", "ifeval:2100", "ifeval:2107", "ifeval:2118", "ifeval:2136", "ifeval:2139", "ifeval:2142", "ifeval:2143", "ifeval:2162", "ifeval:2164", "ifeval:2169", "ifeval:218", "ifeval:2180", "ifeval:219", "ifeval:2192", "ifeval:2195", "ifeval:2207", "ifeval:2209", "ifeval:2215", "ifeval:2216", "ifeval:2225", "ifeval:2229", "ifeval:2230", "ifeval:2239", "ifeval:2243", "ifeval:2245", "ifeval:2246", "ifeval:2247", "ifeval:2250", "ifeval:2253", "ifeval:2265", "ifeval:2266", "ifeval:2268", "ifeval:227", "ifeval:2273", "ifeval:2275", "ifeval:2284", "ifeval:2292", "ifeval:2299", "ifeval:2303", "ifeval:2304", "ifeval:2305", "ifeval:2309", "ifeval:2311", "ifeval:2314", "ifeval:2323", "ifeval:2324", "ifeval:2328", "ifeval:2337", "ifeval:2341", "ifeval:2350", "ifeval:2355", "ifeval:2357", "ifeval:2359", "ifeval:2362", "ifeval:2372", "ifeval:2374", "ifeval:2380", "ifeval:2381", "ifeval:2383", "ifeval:2386", "ifeval:2391", "ifeval:2392", "ifeval:2395", "ifeval:2396", "ifeval:2398", "ifeval:24", "ifeval:240", "ifeval:2404", "ifeval:2416", "ifeval:2417", "ifeval:2422", "ifeval:2432", "ifeval:2436", "ifeval:2439", "ifeval:2441", "ifeval:2447", "ifeval:2449", "ifeval:2457", "ifeval:2464", "ifeval:2465", "ifeval:2467", "ifeval:247", "ifeval:2471", "ifeval:2475", "ifeval:2482", "ifeval:2485", "ifeval:2497", "ifeval:2505", "ifeval:251", "ifeval:2515", "ifeval:2517", "ifeval:2531", "ifeval:2532", "ifeval:2534", "ifeval:2549", "ifeval:2563", "ifeval:2567", "ifeval:2571", "ifeval:2577", "ifeval:2583", "ifeval:2585", "ifeval:2589", "ifeval:2590", "ifeval:2591", "ifeval:2596", "ifeval:260", "ifeval:2602", "ifeval:2605", "ifeval:2616", "ifeval:2617", "ifeval:2622", "ifeval:2628", "ifeval:2637", "ifeval:2649", "ifeval:2653", "ifeval:2662", "ifeval:2667", "ifeval:2674", "ifeval:2677", "ifeval:2683", "ifeval:2685", "ifeval:2691", "ifeval:2704", "ifeval:2713", "ifeval:2716", "ifeval:2724", "ifeval:2728", "ifeval:2736", "ifeval:2739", "ifeval:2749", "ifeval:2751", "ifeval:2752", "ifeval:2759", "ifeval:2765", "ifeval:2768", "ifeval:2779", "ifeval:2780", "ifeval:2785", "ifeval:2787", "ifeval:279", "ifeval:2790", "ifeval:2798", "ifeval:2801", "ifeval:2807", "ifeval:281", "ifeval:2811", "ifeval:2817", "ifeval:2820", "ifeval:2825", "ifeval:2828", "ifeval:2829", "ifeval:2832", "ifeval:2844", "ifeval:2849", "ifeval:2853", "ifeval:2857", "ifeval:2859", "ifeval:286", "ifeval:2870", "ifeval:2871", "ifeval:288", "ifeval:2880", "ifeval:2887", "ifeval:2889", "ifeval:2904", "ifeval:2905", "ifeval:2908", "ifeval:2909", "ifeval:2912", "ifeval:2918", "ifeval:292", "ifeval:2921", "ifeval:2925", "ifeval:2929", "ifeval:2941", "ifeval:2943", "ifeval:295", "ifeval:2957", "ifeval:296", "ifeval:2969", "ifeval:2977", "ifeval:2985", "ifeval:2997", "ifeval:30", "ifeval:3001", "ifeval:301", "ifeval:3025", "ifeval:3035", "ifeval:3041", "ifeval:3048", "ifeval:3057", "ifeval:3063", "ifeval:3069", "ifeval:3071", "ifeval:3073", "ifeval:3079", "ifeval:3081", "ifeval:3084", "ifeval:3089", "ifeval:3091", "ifeval:3098", "ifeval:3109", "ifeval:3112", "ifeval:3114", "ifeval:3126", "ifeval:3130", "ifeval:3131", "ifeval:3150", "ifeval:3156", "ifeval:3166", "ifeval:3186", "ifeval:3188", "ifeval:3191", "ifeval:3195", "ifeval:3198", "ifeval:32", "ifeval:3203", "ifeval:3204", "ifeval:321", "ifeval:322", "ifeval:3221", "ifeval:3223", "ifeval:3224", "ifeval:3241", "ifeval:3243", "ifeval:3245", "ifeval:3256", "ifeval:3263", "ifeval:3272", "ifeval:3276", "ifeval:3280", "ifeval:3281", "ifeval:3287", "ifeval:3294", "ifeval:3305", "ifeval:331", "ifeval:3311", "ifeval:3315", "ifeval:332", "ifeval:3323", "ifeval:3324", "ifeval:3326", "ifeval:3327", "ifeval:3329", "ifeval:3335", "ifeval:334", "ifeval:3345", "ifeval:3350", "ifeval:3351", "ifeval:3362", "ifeval:3367", "ifeval:3369", "ifeval:337", "ifeval:3371", "ifeval:3376", "ifeval:3380", "ifeval:3386", "ifeval:340", "ifeval:3401", "ifeval:3407", "ifeval:3409", "ifeval:3414", "ifeval:3415", "ifeval:3424", "ifeval:3425", "ifeval:3428", "ifeval:3429", "ifeval:343", "ifeval:3434", "ifeval:3439", "ifeval:3442", "ifeval:3445", "ifeval:3453", "ifeval:3455", "ifeval:3456", "ifeval:3478", "ifeval:3479", "ifeval:3484", "ifeval:349", "ifeval:3494", "ifeval:3502", "ifeval:3505", "ifeval:3506", "ifeval:3513", "ifeval:3518", "ifeval:3534", "ifeval:3536", "ifeval:3538", "ifeval:3540", "ifeval:3549", "ifeval:3557", "ifeval:3563", "ifeval:3565", "ifeval:3567", "ifeval:3569", "ifeval:357", "ifeval:3572", "ifeval:3595", "ifeval:3606", "ifeval:3608", "ifeval:3615", "ifeval:3617", "ifeval:3619", "ifeval:3623", "ifeval:3624", "ifeval:3629", "ifeval:3631", "ifeval:3633", "ifeval:3644", "ifeval:3653", "ifeval:3669", "ifeval:3672", "ifeval:3680", "ifeval:3682", "ifeval:3690", "ifeval:3691", "ifeval:3697", "ifeval:3703", "ifeval:3709", "ifeval:371", "ifeval:3710", "ifeval:3718", "ifeval:3719", "ifeval:3724", "ifeval:3732", "ifeval:3739", "ifeval:374", "ifeval:3743", "ifeval:3744", "ifeval:3748", "ifeval:3749", "ifeval:3750", "ifeval:3751", "ifeval:3752", "ifeval:3753", "ifeval:3754", "ifeval:3755", "ifeval:3756", "ifeval:3757" ] }, "hotpot": { "source_urls": [ "https://huggingface.co/datasets/hotpotqa/hotpot_qa" ], "count": 120, "task_ids": [ "hotpot:5a713ea95542994082a3e6e4", "hotpot:5ab659e355429954757d324e", "hotpot:5ac120c55542992a796dedfc", "hotpot:5a7a230e5542996a35c170ee", "hotpot:5ae048a255429924de1b708e", "hotpot:5ae7edee554299540e5a56ad", "hotpot:5a7dd9445542997cc2c474ac", "hotpot:5a8cd381554299653c1aa12b", "hotpot:5add10ca5542994ed6169c59", "hotpot:5ac1bce45542994ab5c67def", "hotpot:5a87954f5542996e4f308856", "hotpot:5a7521c55542993748c897c6", "hotpot:5a713dcb5542994082a3e6d9", "hotpot:5ae7ef625542994a481bbe40", "hotpot:5a74122f55429929fddd83da", "hotpot:5abd0b6b5542996e802b46b4", "hotpot:5a7ed5af5542994959419a84", "hotpot:5ae27f22554299495565da8d", "hotpot:5a8b448255429971feec468c", "hotpot:5a8638945542994775f6072d", "hotpot:5a75fa14554299109176e5dc", "hotpot:5adf88995542993344016cd4", "hotpot:5a74af7e55429916b01641d2", "hotpot:5a7da0915542990b8f5039d4", "hotpot:5ae65c925542992ae0d162f9", "hotpot:5a712e895542994082a3e636", "hotpot:5a8fb3af5542997ba9cb32ee", "hotpot:5ac46bb05542991943173983", "hotpot:5adff2aa554299603e4183d0", "hotpot:5ae063085542993d6555eb37", "hotpot:5abec9b25542997ec76fd366", "hotpot:5a7fb1e55542995d8a8ddef7", "hotpot:5a81d54a55429903bc27b9d4", "hotpot:5a739b1955429978a71e9044", "hotpot:5ab97d0a5542996be202051e", "hotpot:5a8ccea555429941ae14deeb", "hotpot:5ade00855542997dc7907118", "hotpot:5a7933b255429907847277eb", "hotpot:5a79edfe5542990198eaf006", "hotpot:5a7bb16f554299294a54aa9f", "hotpot:5a72160e55429971e9dc9253", "hotpot:5abd1b6e55429933744ab729", "hotpot:5a7e05f75542995f4f402394", "hotpot:5a84abaf5542997b5ce3ff06", "hotpot:5a8434445542996488c2e517", "hotpot:5ac2ada255429967731025f9", "hotpot:5add3a605542990dbb2f7ddb", "hotpot:5ac311925542995ef918c0de", "hotpot:5a88bb4a554299206df2b353", "hotpot:5a7363465542991f29ee2dd5", "hotpot:5a7b241b554299042af8f705", "hotpot:5a84e21d5542997b5ce3ff7c", "hotpot:5ab5eed65542997d4ad1f25c", "hotpot:5a72c9e85542991f9a20c595", "hotpot:5ab81cff5542990e739ec827", "hotpot:5ae20c465542997283cd236a", "hotpot:5a83bfc8554299123d8c21a1", "hotpot:5adc398755429947ff173912", "hotpot:5ab73de755429928e1fe383b", "hotpot:5ae61ac355429929b0807ac6", "hotpot:5ae14def55429901ffe4ae4a", "hotpot:5a7d2b535542995f4f402224", "hotpot:5a7744e45542993569682d37", "hotpot:5a75a76b5542992db9473697", "hotpot:5a8c5d3f554299585d9e3675", "hotpot:5ab250b9554299722f9b4cd9", "hotpot:5a74e6e855429974ef308c8e", "hotpot:5a8d08ee55429941ae14df70", "hotpot:5ae72e5d5542991e8301cba8", "hotpot:5ae1f17e554299234fd04349", "hotpot:5a8c6d4e5542996e8ac88aa6", "hotpot:5ae824295542997ec272772f", "hotpot:5ae0f3b955429920d523424e", "hotpot:5a7b1c6b55429931da12c9ca", "hotpot:5ac00cde554299012d1db58a", "hotpot:5ab5ec2a5542992aa134a3e3", "hotpot:5a74e5aa5542996c70cfae2e", "hotpot:5a883c175542997e5c09a5ae", "hotpot:5ac4f42b55429924173fb50d", "hotpot:5ab3c53955429969a97a81b9", "hotpot:5a835b88554299123d8c2101", "hotpot:5a73024f5542991f9a20c5fc", "hotpot:5a7a0201554299029c4b5f9e", "hotpot:5a7ef6b35542994959419a94", "hotpot:5a778d7855429967ab1051fa", "hotpot:5ae76e615542997ec2727647", "hotpot:5ae2c4d75542991a06ce98d0", "hotpot:5a75f1a755429976ec32bcb1", "hotpot:5a8319a05542990548d0b178", "hotpot:5a8b97485542995d1e6f140f", "hotpot:5a811667554299260e20a23d", "hotpot:5ab30336554299166977417c", "hotpot:5a8c2d23554299240d9c20b7", "hotpot:5ae492955542995ad6573daa", "hotpot:5a83161555429940e5e1a958", "hotpot:5a887771554299206df2b294", "hotpot:5ab4396555429942dd415eed", "hotpot:5ab85e195542992aa3b8c8c5", "hotpot:5a8e7c095542990e94052b22", "hotpot:5aba9813554299232ef4a357", "hotpot:5a78e86f55429974737f78fb", "hotpot:5ab5cc4a554299637185c61b", "hotpot:5a85a37d5542997175ce1fe5", "hotpot:5abb14d85542996cc5e49f92", "hotpot:5ae1ab2f554299234fd042ba", "hotpot:5abd848455429924427fd03a", "hotpot:5ac2dc4b55429921a00ab083", "hotpot:5a762d53554299109176e6a7", "hotpot:5a8c9641554299585d9e36f5", "hotpot:5a85f2f55542997b5ce40036", "hotpot:5a7fc6645542992e7d278d6e", "hotpot:5a8900c75542997e5c09a6ed", "hotpot:5a746a4455429974ef308bf6", "hotpot:5a8e612b5542995085b3741b", "hotpot:5ae1054055429920d5234284", "hotpot:5a79f37a5542996a35c17045", "hotpot:5a8f84505542992414482aef", "hotpot:5a84c6df5542997b5ce3ff61", "hotpot:5a7fd7a25542992e7d278dc3", "hotpot:5a7776b955429967ab10519e" ] }, "cz_cermat": { "source_urls": [ "https://huggingface.co/datasets/CZLC/cermat_czech_mc" ], "count": 649, "task_ids": [ "cz_cermat:0000", "cz_cermat:0001", "cz_cermat:0002", "cz_cermat:0003", "cz_cermat:0004", "cz_cermat:0005", "cz_cermat:0006", "cz_cermat:0007", "cz_cermat:0008", "cz_cermat:0009", "cz_cermat:0010", "cz_cermat:0011", "cz_cermat:0012", "cz_cermat:0013", "cz_cermat:0014", "cz_cermat:0015", "cz_cermat:0016", "cz_cermat:0017", "cz_cermat:0018", "cz_cermat:0019", "cz_cermat:0020", "cz_cermat:0021", "cz_cermat:0022", "cz_cermat:0023", "cz_cermat:0024", "cz_cermat:0025", "cz_cermat:0026", "cz_cermat:0027", "cz_cermat:0028", "cz_cermat:0029", "cz_cermat:0030", "cz_cermat:0031", "cz_cermat:0032", "cz_cermat:0033", "cz_cermat:0034", "cz_cermat:0035", "cz_cermat:0036", "cz_cermat:0037", "cz_cermat:0038", "cz_cermat:0039", "cz_cermat:0040", "cz_cermat:0041", "cz_cermat:0042", "cz_cermat:0043", "cz_cermat:0044", "cz_cermat:0045", "cz_cermat:0046", "cz_cermat:0047", "cz_cermat:0048", "cz_cermat:0049", "cz_cermat:0050", "cz_cermat:0051", "cz_cermat:0052", "cz_cermat:0053", "cz_cermat:0054", "cz_cermat:0055", "cz_cermat:0056", "cz_cermat:0057", "cz_cermat:0058", "cz_cermat:0059", "cz_cermat:0060", "cz_cermat:0061", "cz_cermat:0062", "cz_cermat:0063", "cz_cermat:0064", "cz_cermat:0065", "cz_cermat:0066", "cz_cermat:0067", "cz_cermat:0068", "cz_cermat:0069", "cz_cermat:0070", "cz_cermat:0071", "cz_cermat:0072", "cz_cermat:0073", "cz_cermat:0074", "cz_cermat:0075", "cz_cermat:0076", "cz_cermat:0077", "cz_cermat:0078", "cz_cermat:0079", "cz_cermat:0080", "cz_cermat:0081", "cz_cermat:0082", "cz_cermat:0083", "cz_cermat:0084", "cz_cermat:0085", "cz_cermat:0086", "cz_cermat:0087", "cz_cermat:0088", "cz_cermat:0089", "cz_cermat:0090", "cz_cermat:0091", "cz_cermat:0092", "cz_cermat:0093", "cz_cermat:0094", "cz_cermat:0095", "cz_cermat:0096", "cz_cermat:0097", "cz_cermat:0098", "cz_cermat:0099", "cz_cermat:0100", "cz_cermat:0101", "cz_cermat:0102", "cz_cermat:0103", "cz_cermat:0104", "cz_cermat:0105", "cz_cermat:0106", "cz_cermat:0107", "cz_cermat:0108", "cz_cermat:0109", "cz_cermat:0110", "cz_cermat:0111", "cz_cermat:0112", "cz_cermat:0113", "cz_cermat:0114", "cz_cermat:0115", "cz_cermat:0116", "cz_cermat:0117", "cz_cermat:0118", "cz_cermat:0119", "cz_cermat:0120", "cz_cermat:0121", "cz_cermat:0122", "cz_cermat:0123", "cz_cermat:0124", "cz_cermat:0125", "cz_cermat:0126", "cz_cermat:0127", "cz_cermat:0128", "cz_cermat:0129", "cz_cermat:0130", "cz_cermat:0131", "cz_cermat:0132", "cz_cermat:0133", "cz_cermat:0134", "cz_cermat:0135", "cz_cermat:0136", "cz_cermat:0137", "cz_cermat:0138", "cz_cermat:0139", "cz_cermat:0140", "cz_cermat:0141", "cz_cermat:0142", "cz_cermat:0143", "cz_cermat:0144", "cz_cermat:0145", "cz_cermat:0146", "cz_cermat:0147", "cz_cermat:0148", "cz_cermat:0149", "cz_cermat:0150", "cz_cermat:0151", "cz_cermat:0152", "cz_cermat:0153", "cz_cermat:0154", "cz_cermat:0155", "cz_cermat:0156", "cz_cermat:0157", "cz_cermat:0158", "cz_cermat:0159", "cz_cermat:0160", "cz_cermat:0161", "cz_cermat:0162", "cz_cermat:0163", "cz_cermat:0164", "cz_cermat:0165", "cz_cermat:0166", "cz_cermat:0167", "cz_cermat:0168", "cz_cermat:0169", "cz_cermat:0170", "cz_cermat:0171", "cz_cermat:0172", "cz_cermat:0173", "cz_cermat:0174", "cz_cermat:0175", "cz_cermat:0176", "cz_cermat:0177", "cz_cermat:0178", "cz_cermat:0179", "cz_cermat:0180", "cz_cermat:0181", "cz_cermat:0182", "cz_cermat:0183", "cz_cermat:0184", "cz_cermat:0185", "cz_cermat:0186", "cz_cermat:0187", "cz_cermat:0188", "cz_cermat:0189", "cz_cermat:0190", "cz_cermat:0191", "cz_cermat:0192", "cz_cermat:0193", "cz_cermat:0194", "cz_cermat:0195", "cz_cermat:0196", "cz_cermat:0197", "cz_cermat:0198", "cz_cermat:0199", "cz_cermat:0200", "cz_cermat:0201", "cz_cermat:0202", "cz_cermat:0203", "cz_cermat:0204", "cz_cermat:0205", "cz_cermat:0206", "cz_cermat:0207", "cz_cermat:0208", "cz_cermat:0209", "cz_cermat:0210", "cz_cermat:0211", "cz_cermat:0212", "cz_cermat:0213", "cz_cermat:0214", "cz_cermat:0215", "cz_cermat:0216", "cz_cermat:0217", "cz_cermat:0218", "cz_cermat:0219", "cz_cermat:0220", "cz_cermat:0221", "cz_cermat:0222", "cz_cermat:0223", "cz_cermat:0224", "cz_cermat:0225", "cz_cermat:0226", "cz_cermat:0227", "cz_cermat:0228", "cz_cermat:0229", "cz_cermat:0230", "cz_cermat:0231", "cz_cermat:0232", "cz_cermat:0233", "cz_cermat:0234", "cz_cermat:0235", "cz_cermat:0236", "cz_cermat:0237", "cz_cermat:0238", "cz_cermat:0239", "cz_cermat:0240", "cz_cermat:0241", "cz_cermat:0242", "cz_cermat:0243", "cz_cermat:0244", "cz_cermat:0245", "cz_cermat:0246", "cz_cermat:0247", "cz_cermat:0248", "cz_cermat:0249", "cz_cermat:0250", "cz_cermat:0251", "cz_cermat:0252", "cz_cermat:0253", "cz_cermat:0254", "cz_cermat:0255", "cz_cermat:0256", "cz_cermat:0257", "cz_cermat:0258", "cz_cermat:0259", "cz_cermat:0260", "cz_cermat:0261", "cz_cermat:0262", "cz_cermat:0263", "cz_cermat:0264", "cz_cermat:0265", "cz_cermat:0266", "cz_cermat:0267", "cz_cermat:0268", "cz_cermat:0269", "cz_cermat:0270", "cz_cermat:0271", "cz_cermat:0272", "cz_cermat:0273", "cz_cermat:0274", "cz_cermat:0275", "cz_cermat:0276", "cz_cermat:0277", "cz_cermat:0278", "cz_cermat:0279", "cz_cermat:0280", "cz_cermat:0281", "cz_cermat:0282", "cz_cermat:0283", "cz_cermat:0284", "cz_cermat:0285", "cz_cermat:0286", "cz_cermat:0287", "cz_cermat:0288", "cz_cermat:0289", "cz_cermat:0290", "cz_cermat:0291", "cz_cermat:0292", "cz_cermat:0293", "cz_cermat:0294", "cz_cermat:0295", "cz_cermat:0296", "cz_cermat:0297", "cz_cermat:0298", "cz_cermat:0299", "cz_cermat:0300", "cz_cermat:0301", "cz_cermat:0302", "cz_cermat:0303", "cz_cermat:0304", "cz_cermat:0305", "cz_cermat:0306", "cz_cermat:0307", "cz_cermat:0308", "cz_cermat:0309", "cz_cermat:0310", "cz_cermat:0311", "cz_cermat:0312", "cz_cermat:0313", "cz_cermat:0314", "cz_cermat:0315", "cz_cermat:0316", "cz_cermat:0317", "cz_cermat:0318", "cz_cermat:0319", "cz_cermat:0320", "cz_cermat:0321", "cz_cermat:0322", "cz_cermat:0323", "cz_cermat:0324", "cz_cermat:0325", "cz_cermat:0326", "cz_cermat:0327", "cz_cermat:0328", "cz_cermat:0329", "cz_cermat:0330", "cz_cermat:0331", "cz_cermat:0332", "cz_cermat:0333", "cz_cermat:0334", "cz_cermat:0335", "cz_cermat:0336", "cz_cermat:0337", "cz_cermat:0338", "cz_cermat:0339", "cz_cermat:0340", "cz_cermat:0341", "cz_cermat:0342", "cz_cermat:0343", "cz_cermat:0344", "cz_cermat:0345", "cz_cermat:0346", "cz_cermat:0347", "cz_cermat:0348", "cz_cermat:0349", "cz_cermat:0350", "cz_cermat:0351", "cz_cermat:0352", "cz_cermat:0353", "cz_cermat:0354", "cz_cermat:0355", "cz_cermat:0356", "cz_cermat:0357", "cz_cermat:0358", "cz_cermat:0359", "cz_cermat:0360", "cz_cermat:0361", "cz_cermat:0362", "cz_cermat:0363", "cz_cermat:0364", "cz_cermat:0365", "cz_cermat:0366", "cz_cermat:0367", "cz_cermat:0368", "cz_cermat:0369", "cz_cermat:0370", "cz_cermat:0371", "cz_cermat:0372", "cz_cermat:0373", "cz_cermat:0374", "cz_cermat:0375", "cz_cermat:0376", "cz_cermat:0377", "cz_cermat:0378", "cz_cermat:0379", "cz_cermat:0380", "cz_cermat:0381", "cz_cermat:0382", "cz_cermat:0383", "cz_cermat:0384", "cz_cermat:0385", "cz_cermat:0386", "cz_cermat:0387", "cz_cermat:0388", "cz_cermat:0389", "cz_cermat:0390", "cz_cermat:0391", "cz_cermat:0392", "cz_cermat:0393", "cz_cermat:0394", "cz_cermat:0395", "cz_cermat:0396", "cz_cermat:0397", "cz_cermat:0398", "cz_cermat:0399", "cz_cermat:0400", "cz_cermat:0401", "cz_cermat:0402", "cz_cermat:0403", "cz_cermat:0404", "cz_cermat:0405", "cz_cermat:0406", "cz_cermat:0407", "cz_cermat:0408", "cz_cermat:0409", "cz_cermat:0410", "cz_cermat:0411", "cz_cermat:0412", "cz_cermat:0413", "cz_cermat:0414", "cz_cermat:0415", "cz_cermat:0416", "cz_cermat:0417", "cz_cermat:0418", "cz_cermat:0419", "cz_cermat:0420", "cz_cermat:0421", "cz_cermat:0422", "cz_cermat:0423", "cz_cermat:0424", "cz_cermat:0425", "cz_cermat:0426", "cz_cermat:0427", "cz_cermat:0428", "cz_cermat:0429", "cz_cermat:0430", "cz_cermat:0431", "cz_cermat:0432", "cz_cermat:0433", "cz_cermat:0434", "cz_cermat:0435", "cz_cermat:0436", "cz_cermat:0437", "cz_cermat:0438", "cz_cermat:0439", "cz_cermat:0440", "cz_cermat:0441", "cz_cermat:0442", "cz_cermat:0443", "cz_cermat:0444", "cz_cermat:0445", "cz_cermat:0446", "cz_cermat:0447", "cz_cermat:0448", "cz_cermat:0449", "cz_cermat:0450", "cz_cermat:0451", "cz_cermat:0452", "cz_cermat:0453", "cz_cermat:0454", "cz_cermat:0455", "cz_cermat:0456", "cz_cermat:0457", "cz_cermat:0458", "cz_cermat:0459", "cz_cermat:0460", "cz_cermat:0461", "cz_cermat:0462", "cz_cermat:0463", "cz_cermat:0464", "cz_cermat:0465", "cz_cermat:0466", "cz_cermat:0467", "cz_cermat:0468", "cz_cermat:0469", "cz_cermat:0470", "cz_cermat:0471", "cz_cermat:0472", "cz_cermat:0473", "cz_cermat:0474", "cz_cermat:0475", "cz_cermat:0476", "cz_cermat:0477", "cz_cermat:0478", "cz_cermat:0479", "cz_cermat:0480", "cz_cermat:0481", "cz_cermat:0482", "cz_cermat:0483", "cz_cermat:0484", "cz_cermat:0485", "cz_cermat:0486", "cz_cermat:0487", "cz_cermat:0488", "cz_cermat:0489", "cz_cermat:0490", "cz_cermat:0491", "cz_cermat:0492", "cz_cermat:0493", "cz_cermat:0494", "cz_cermat:0495", "cz_cermat:0496", "cz_cermat:0497", "cz_cermat:0498", "cz_cermat:0499", "cz_cermat:0500", "cz_cermat:0501", "cz_cermat:0502", "cz_cermat:0503", "cz_cermat:0504", "cz_cermat:0505", "cz_cermat:0506", "cz_cermat:0507", "cz_cermat:0508", "cz_cermat:0509", "cz_cermat:0510", "cz_cermat:0511", "cz_cermat:0512", "cz_cermat:0513", "cz_cermat:0514", "cz_cermat:0515", "cz_cermat:0516", "cz_cermat:0517", "cz_cermat:0518", "cz_cermat:0519", "cz_cermat:0520", "cz_cermat:0521", "cz_cermat:0522", "cz_cermat:0523", "cz_cermat:0524", "cz_cermat:0525", "cz_cermat:0526", "cz_cermat:0527", "cz_cermat:0528", "cz_cermat:0529", "cz_cermat:0530", "cz_cermat:0531", "cz_cermat:0532", "cz_cermat:0533", "cz_cermat:0534", "cz_cermat:0535", "cz_cermat:0536", "cz_cermat:0537", "cz_cermat:0538", "cz_cermat:0539", "cz_cermat:0540", "cz_cermat:0541", "cz_cermat:0542", "cz_cermat:0543", "cz_cermat:0544", "cz_cermat:0545", "cz_cermat:0546", "cz_cermat:0547", "cz_cermat:0548", "cz_cermat:0549", "cz_cermat:0550", "cz_cermat:0551", "cz_cermat:0552", "cz_cermat:0553", "cz_cermat:0554", "cz_cermat:0555", "cz_cermat:0556", "cz_cermat:0557", "cz_cermat:0558", "cz_cermat:0559", "cz_cermat:0560", "cz_cermat:0561", "cz_cermat:0562", "cz_cermat:0563", "cz_cermat:0564", "cz_cermat:0565", "cz_cermat:0566", "cz_cermat:0567", "cz_cermat:0568", "cz_cermat:0569", "cz_cermat:0570", "cz_cermat:0571", "cz_cermat:0572", "cz_cermat:0573", "cz_cermat:0574", "cz_cermat:0575", "cz_cermat:0576", "cz_cermat:0577", "cz_cermat:0578", "cz_cermat:0579", "cz_cermat:0580", "cz_cermat:0581", "cz_cermat:0582", "cz_cermat:0583", "cz_cermat:0584", "cz_cermat:0585", "cz_cermat:0586", "cz_cermat:0587", "cz_cermat:0588", "cz_cermat:0589", "cz_cermat:0590", "cz_cermat:0591", "cz_cermat:0592", "cz_cermat:0593", "cz_cermat:0594", "cz_cermat:0595", "cz_cermat:0596", "cz_cermat:0597", "cz_cermat:0598", "cz_cermat:0599", "cz_cermat:0600", "cz_cermat:0601", "cz_cermat:0602", "cz_cermat:0603", "cz_cermat:0604", "cz_cermat:0605", "cz_cermat:0606", "cz_cermat:0607", "cz_cermat:0608", "cz_cermat:0609", "cz_cermat:0610", "cz_cermat:0611", "cz_cermat:0612", "cz_cermat:0613", "cz_cermat:0614", "cz_cermat:0615", "cz_cermat:0616", "cz_cermat:0617", "cz_cermat:0618", "cz_cermat:0619", "cz_cermat:0620", "cz_cermat:0621", "cz_cermat:0622", "cz_cermat:0623", "cz_cermat:0624", "cz_cermat:0625", "cz_cermat:0626", "cz_cermat:0627", "cz_cermat:0628", "cz_cermat:0629", "cz_cermat:0630", "cz_cermat:0631", "cz_cermat:0632", "cz_cermat:0633", "cz_cermat:0634", "cz_cermat:0635", "cz_cermat:0636", "cz_cermat:0637", "cz_cermat:0638", "cz_cermat:0639", "cz_cermat:0640", "cz_cermat:0641", "cz_cermat:0642", "cz_cermat:0643", "cz_cermat:0644", "cz_cermat:0645", "cz_cermat:0646", "cz_cermat:0647", "cz_cermat:0648" ] }, "cz_math": { "source_urls": [ "https://huggingface.co/datasets/CZLC/cermat_math_mc" ], "count": 126, "task_ids": [ "cz_math:0000", "cz_math:0001", "cz_math:0002", "cz_math:0003", "cz_math:0004", "cz_math:0005", "cz_math:0006", "cz_math:0007", "cz_math:0008", "cz_math:0009", "cz_math:0010", "cz_math:0011", "cz_math:0012", "cz_math:0013", "cz_math:0014", "cz_math:0015", "cz_math:0016", "cz_math:0017", "cz_math:0018", "cz_math:0019", "cz_math:0020", "cz_math:0021", "cz_math:0022", "cz_math:0023", "cz_math:0024", "cz_math:0025", "cz_math:0026", "cz_math:0027", "cz_math:0028", "cz_math:0029", "cz_math:0030", "cz_math:0031", "cz_math:0032", "cz_math:0033", "cz_math:0034", "cz_math:0035", "cz_math:0036", "cz_math:0037", "cz_math:0038", "cz_math:0039", "cz_math:0040", "cz_math:0041", "cz_math:0042", "cz_math:0043", "cz_math:0044", "cz_math:0045", "cz_math:0046", "cz_math:0047", "cz_math:0048", "cz_math:0049", "cz_math:0050", "cz_math:0051", "cz_math:0052", "cz_math:0053", "cz_math:0054", "cz_math:0055", "cz_math:0056", "cz_math:0057", "cz_math:0058", "cz_math:0059", "cz_math:0060", "cz_math:0061", "cz_math:0062", "cz_math:0063", "cz_math:0064", "cz_math:0065", "cz_math:0066", "cz_math:0067", "cz_math:0068", "cz_math:0069", "cz_math:0070", "cz_math:0071", "cz_math:0072", "cz_math:0073", "cz_math:0074", "cz_math:0075", "cz_math:0076", "cz_math:0077", "cz_math:0078", "cz_math:0079", "cz_math:0080", "cz_math:0081", "cz_math:0082", "cz_math:0083", "cz_math:0084", "cz_math:0085", "cz_math:0086", "cz_math:0087", "cz_math:0088", "cz_math:0089", "cz_math:0090", "cz_math:0091", "cz_math:0092", "cz_math:0093", "cz_math:0094", "cz_math:0095", "cz_math:0096", "cz_math:0097", "cz_math:0098", "cz_math:0099", "cz_math:0100", "cz_math:0101", "cz_math:0102", "cz_math:0103", "cz_math:0104", "cz_math:0105", "cz_math:0106", "cz_math:0107", "cz_math:0108", "cz_math:0109", "cz_math:0110", "cz_math:0111", "cz_math:0112", "cz_math:0113", "cz_math:0114", "cz_math:0115", "cz_math:0116", "cz_math:0117", "cz_math:0118", "cz_math:0119", "cz_math:0120", "cz_math:0121", "cz_math:0122", "cz_math:0123", "cz_math:0124", "cz_math:0125" ] }, "cz_mmlu": { "source_urls": [ "https://huggingface.co/datasets/CIIRC-NLP/mmlu-cs" ], "count": 290, "task_ids": [ "cz_mmlu:abstract_algebra:3", "cz_mmlu:abstract_algebra:14", "cz_mmlu:abstract_algebra:35", "cz_mmlu:abstract_algebra:81", "cz_mmlu:abstract_algebra:94", "cz_mmlu:anatomy:22", "cz_mmlu:anatomy:26", "cz_mmlu:anatomy:35", "cz_mmlu:anatomy:57", "cz_mmlu:anatomy:62", "cz_mmlu:astronomy:7", "cz_mmlu:astronomy:8", "cz_mmlu:astronomy:23", "cz_mmlu:astronomy:108", "cz_mmlu:astronomy:151", "cz_mmlu:business_ethics:3", "cz_mmlu:business_ethics:27", "cz_mmlu:business_ethics:29", "cz_mmlu:business_ethics:64", "cz_mmlu:business_ethics:77", "cz_mmlu:clinical_knowledge:101", "cz_mmlu:clinical_knowledge:112", "cz_mmlu:clinical_knowledge:142", "cz_mmlu:clinical_knowledge:214", "cz_mmlu:clinical_knowledge:229", "cz_mmlu:college_biology:1", "cz_mmlu:college_biology:40", "cz_mmlu:college_biology:71", "cz_mmlu:college_biology:87", "cz_mmlu:college_biology:108", "cz_mmlu:college_chemistry:13", "cz_mmlu:college_chemistry:19", "cz_mmlu:college_chemistry:27", "cz_mmlu:college_chemistry:43", "cz_mmlu:college_chemistry:97", "cz_mmlu:college_computer_science:11", "cz_mmlu:college_computer_science:12", "cz_mmlu:college_computer_science:44", "cz_mmlu:college_computer_science:45", "cz_mmlu:college_computer_science:48", "cz_mmlu:college_mathematics:5", "cz_mmlu:college_mathematics:33", "cz_mmlu:college_mathematics:58", "cz_mmlu:college_mathematics:77", "cz_mmlu:college_mathematics:93", "cz_mmlu:college_medicine:20", "cz_mmlu:college_medicine:31", "cz_mmlu:college_medicine:96", "cz_mmlu:college_medicine:137", "cz_mmlu:college_medicine:141", "cz_mmlu:college_physics:37", "cz_mmlu:college_physics:46", "cz_mmlu:college_physics:73", "cz_mmlu:college_physics:79", "cz_mmlu:college_physics:80", "cz_mmlu:computer_security:5", "cz_mmlu:computer_security:8", "cz_mmlu:computer_security:24", "cz_mmlu:computer_security:84", "cz_mmlu:computer_security:90", "cz_mmlu:conceptual_physics:20", "cz_mmlu:conceptual_physics:58", "cz_mmlu:conceptual_physics:74", "cz_mmlu:conceptual_physics:197", "cz_mmlu:conceptual_physics:218", "cz_mmlu:data:1654", "cz_mmlu:data:3814", "cz_mmlu:data:4554", "cz_mmlu:data:6227", "cz_mmlu:data:7428", "cz_mmlu:econometrics:20", "cz_mmlu:econometrics:46", "cz_mmlu:econometrics:47", "cz_mmlu:econometrics:81", "cz_mmlu:econometrics:106", "cz_mmlu:electrical_engineering:18", "cz_mmlu:electrical_engineering:43", "cz_mmlu:electrical_engineering:53", "cz_mmlu:electrical_engineering:68", "cz_mmlu:electrical_engineering:90", "cz_mmlu:elementary_mathematics:83", "cz_mmlu:elementary_mathematics:125", "cz_mmlu:elementary_mathematics:236", "cz_mmlu:elementary_mathematics:273", "cz_mmlu:elementary_mathematics:373", "cz_mmlu:formal_logic:34", "cz_mmlu:formal_logic:48", "cz_mmlu:formal_logic:81", "cz_mmlu:formal_logic:88", "cz_mmlu:formal_logic:118", "cz_mmlu:global_facts:28", "cz_mmlu:global_facts:41", "cz_mmlu:global_facts:71", "cz_mmlu:global_facts:87", "cz_mmlu:global_facts:98", "cz_mmlu:high_school_biology:16", "cz_mmlu:high_school_biology:28", "cz_mmlu:high_school_biology:117", "cz_mmlu:high_school_biology:161", "cz_mmlu:high_school_biology:205", "cz_mmlu:high_school_chemistry:16", "cz_mmlu:high_school_chemistry:54", "cz_mmlu:high_school_chemistry:68", "cz_mmlu:high_school_chemistry:145", "cz_mmlu:high_school_chemistry:183", "cz_mmlu:high_school_computer_science:27", "cz_mmlu:high_school_computer_science:40", "cz_mmlu:high_school_computer_science:50", "cz_mmlu:high_school_computer_science:63", "cz_mmlu:high_school_computer_science:83", "cz_mmlu:high_school_european_history:35", "cz_mmlu:high_school_european_history:36", "cz_mmlu:high_school_european_history:67", "cz_mmlu:high_school_european_history:117", "cz_mmlu:high_school_european_history:164", "cz_mmlu:high_school_geography:63", "cz_mmlu:high_school_geography:67", "cz_mmlu:high_school_geography:137", "cz_mmlu:high_school_geography:143", "cz_mmlu:high_school_geography:190", "cz_mmlu:high_school_government_and_politics:92", "cz_mmlu:high_school_government_and_politics:102", "cz_mmlu:high_school_government_and_politics:109", "cz_mmlu:high_school_government_and_politics:149", "cz_mmlu:high_school_government_and_politics:191", "cz_mmlu:high_school_macroeconomics:46", "cz_mmlu:high_school_macroeconomics:70", "cz_mmlu:high_school_macroeconomics:112", "cz_mmlu:high_school_macroeconomics:252", "cz_mmlu:high_school_macroeconomics:260", "cz_mmlu:high_school_mathematics:24", "cz_mmlu:high_school_mathematics:56", "cz_mmlu:high_school_mathematics:78", "cz_mmlu:high_school_mathematics:81", "cz_mmlu:high_school_mathematics:216", "cz_mmlu:high_school_microeconomics:16", "cz_mmlu:high_school_microeconomics:97", "cz_mmlu:high_school_microeconomics:98", "cz_mmlu:high_school_microeconomics:119", "cz_mmlu:high_school_microeconomics:152", "cz_mmlu:high_school_physics:2", "cz_mmlu:high_school_physics:29", "cz_mmlu:high_school_physics:64", "cz_mmlu:high_school_physics:135", "cz_mmlu:high_school_physics:141", "cz_mmlu:high_school_psychology:114", "cz_mmlu:high_school_psychology:273", "cz_mmlu:high_school_psychology:300", "cz_mmlu:high_school_psychology:348", "cz_mmlu:high_school_psychology:445", "cz_mmlu:high_school_statistics:0", "cz_mmlu:high_school_statistics:40", "cz_mmlu:high_school_statistics:67", "cz_mmlu:high_school_statistics:116", "cz_mmlu:high_school_statistics:184", "cz_mmlu:high_school_us_history:27", "cz_mmlu:high_school_us_history:45", "cz_mmlu:high_school_us_history:128", "cz_mmlu:high_school_us_history:129", "cz_mmlu:high_school_us_history:195", "cz_mmlu:high_school_world_history:76", "cz_mmlu:high_school_world_history:160", "cz_mmlu:high_school_world_history:163", "cz_mmlu:high_school_world_history:215", "cz_mmlu:high_school_world_history:222", "cz_mmlu:human_aging:39", "cz_mmlu:human_aging:50", "cz_mmlu:human_aging:95", "cz_mmlu:human_aging:129", "cz_mmlu:human_aging:155", "cz_mmlu:human_sexuality:0", "cz_mmlu:human_sexuality:4", "cz_mmlu:human_sexuality:41", "cz_mmlu:human_sexuality:82", "cz_mmlu:human_sexuality:125", "cz_mmlu:international_law:14", "cz_mmlu:international_law:46", "cz_mmlu:international_law:106", "cz_mmlu:international_law:112", "cz_mmlu:international_law:118", "cz_mmlu:jurisprudence:7", "cz_mmlu:jurisprudence:30", "cz_mmlu:jurisprudence:39", "cz_mmlu:jurisprudence:72", "cz_mmlu:jurisprudence:103", "cz_mmlu:logical_fallacies:17", "cz_mmlu:logical_fallacies:20", "cz_mmlu:logical_fallacies:21", "cz_mmlu:logical_fallacies:124", "cz_mmlu:logical_fallacies:136", "cz_mmlu:machine_learning:16", "cz_mmlu:machine_learning:60", "cz_mmlu:machine_learning:70", "cz_mmlu:machine_learning:84", "cz_mmlu:machine_learning:98", "cz_mmlu:management:21", "cz_mmlu:management:33", "cz_mmlu:management:54", "cz_mmlu:management:67", "cz_mmlu:management:77", "cz_mmlu:marketing:54", "cz_mmlu:marketing:138", "cz_mmlu:marketing:176", "cz_mmlu:marketing:186", "cz_mmlu:marketing:193", "cz_mmlu:medical_genetics:25", "cz_mmlu:medical_genetics:39", "cz_mmlu:medical_genetics:51", "cz_mmlu:medical_genetics:85", "cz_mmlu:medical_genetics:91", "cz_mmlu:miscellaneous:382", "cz_mmlu:miscellaneous:448", "cz_mmlu:miscellaneous:462", "cz_mmlu:miscellaneous:529", "cz_mmlu:miscellaneous:665", "cz_mmlu:moral_disputes:32", "cz_mmlu:moral_disputes:61", "cz_mmlu:moral_disputes:115", "cz_mmlu:moral_disputes:126", "cz_mmlu:moral_disputes:173", "cz_mmlu:moral_scenarios:21", "cz_mmlu:moral_scenarios:225", "cz_mmlu:moral_scenarios:235", "cz_mmlu:moral_scenarios:567", "cz_mmlu:moral_scenarios:602", "cz_mmlu:nutrition:3", "cz_mmlu:nutrition:30", "cz_mmlu:nutrition:34", "cz_mmlu:nutrition:36", "cz_mmlu:nutrition:117", "cz_mmlu:philosophy:16", "cz_mmlu:philosophy:36", "cz_mmlu:philosophy:121", "cz_mmlu:philosophy:169", "cz_mmlu:philosophy:263", "cz_mmlu:prehistory:67", "cz_mmlu:prehistory:109", "cz_mmlu:prehistory:142", "cz_mmlu:prehistory:248", "cz_mmlu:prehistory:276", "cz_mmlu:professional_accounting:48", "cz_mmlu:professional_accounting:97", "cz_mmlu:professional_accounting:124", "cz_mmlu:professional_accounting:208", "cz_mmlu:professional_accounting:242", "cz_mmlu:professional_law:198", "cz_mmlu:professional_law:725", "cz_mmlu:professional_law:867", "cz_mmlu:professional_law:882", "cz_mmlu:professional_law:1349", "cz_mmlu:professional_medicine:27", "cz_mmlu:professional_medicine:31", "cz_mmlu:professional_medicine:50", "cz_mmlu:professional_medicine:210", "cz_mmlu:professional_medicine:239", "cz_mmlu:professional_psychology:111", "cz_mmlu:professional_psychology:196", "cz_mmlu:professional_psychology:254", "cz_mmlu:professional_psychology:347", "cz_mmlu:professional_psychology:412", "cz_mmlu:public_relations:17", "cz_mmlu:public_relations:24", "cz_mmlu:public_relations:54", "cz_mmlu:public_relations:57", "cz_mmlu:public_relations:68", "cz_mmlu:security_studies:46", "cz_mmlu:security_studies:63", "cz_mmlu:security_studies:71", "cz_mmlu:security_studies:118", "cz_mmlu:security_studies:223", "cz_mmlu:sociology:12", "cz_mmlu:sociology:19", "cz_mmlu:sociology:25", "cz_mmlu:sociology:113", "cz_mmlu:sociology:140", "cz_mmlu:us_foreign_policy:1", "cz_mmlu:us_foreign_policy:11", "cz_mmlu:us_foreign_policy:69", "cz_mmlu:us_foreign_policy:83", "cz_mmlu:us_foreign_policy:96", "cz_mmlu:virology:42", "cz_mmlu:virology:60", "cz_mmlu:virology:104", "cz_mmlu:virology:123", "cz_mmlu:virology:124", "cz_mmlu:world_religions:15", "cz_mmlu:world_religions:42", "cz_mmlu:world_religions:54", "cz_mmlu:world_religions:97", "cz_mmlu:world_religions:102" ] }, "lcb": { "source_urls": [ "https://huggingface.co/datasets/livecodebench/code_generation_lite" ], "count": 60, "task_ids": [ "lcb:abc395_c", "lcb:abc395_f", "lcb:abc395_b", "lcb:abc395_a", "lcb:abc395_e", "lcb:abc396_g", "lcb:abc396_c", "lcb:abc396_f", "lcb:abc396_b", "lcb:abc396_d", "lcb:abc396_a", "lcb:abc397_d", "lcb:abc397_f", "lcb:abc397_c", "lcb:abc397_g", "lcb:abc397_a", "lcb:abc397_b", "lcb:abc398_f", "lcb:abc398_b", "lcb:abc398_d", "lcb:abc398_a", "lcb:abc398_c", "lcb:abc399_e", "lcb:abc399_d", "lcb:abc399_a", "lcb:abc399_c", "lcb:abc399_f", "lcb:abc399_b", "lcb:abc400_c", "lcb:abc400_d", "lcb:abc400_a", "lcb:abc400_b", "lcb:abc400_e", "lcb:abc400_g", "lcb:arc194_a", "lcb:arc194_d", "lcb:arc195_c", "lcb:arc196_b", "lcb:arc196_c", "lcb:arc196_d", "lcb:3705", "lcb:3722", "lcb:3770", "lcb:3759", "lcb:3791", "lcb:3789", "lcb:3788", "lcb:3750", "lcb:3795", "lcb:3817", "lcb:3809", "lcb:3832", "lcb:3793", "lcb:3773", "lcb:3778", "lcb:3785", "lcb:3799", "lcb:3811", "lcb:3805", "lcb:3765" ] }, "bbh": { "source_urls": [ "https://huggingface.co/datasets/maveriq/bigbenchhard" ], "count": 405, "task_ids": [ "bbh:boolean_expressions:6", "bbh:boolean_expressions:22", "bbh:boolean_expressions:26", "bbh:boolean_expressions:28", "bbh:boolean_expressions:35", "bbh:boolean_expressions:57", "bbh:boolean_expressions:62", "bbh:boolean_expressions:70", "bbh:boolean_expressions:139", "bbh:boolean_expressions:151", "bbh:boolean_expressions:163", "bbh:boolean_expressions:173", "bbh:boolean_expressions:188", "bbh:boolean_expressions:189", "bbh:boolean_expressions:228", "bbh:causal_judgement:6", "bbh:causal_judgement:7", "bbh:causal_judgement:8", "bbh:causal_judgement:23", "bbh:causal_judgement:50", "bbh:causal_judgement:55", "bbh:causal_judgement:59", "bbh:causal_judgement:108", "bbh:causal_judgement:129", "bbh:causal_judgement:139", "bbh:causal_judgement:143", "bbh:causal_judgement:154", "bbh:causal_judgement:166", "bbh:causal_judgement:179", "bbh:causal_judgement:183", "bbh:date_understanding:1", "bbh:date_understanding:39", "bbh:date_understanding:40", "bbh:date_understanding:56", "bbh:date_understanding:71", "bbh:date_understanding:87", "bbh:date_understanding:107", "bbh:date_understanding:108", "bbh:date_understanding:114", "bbh:date_understanding:150", "bbh:date_understanding:178", "bbh:date_understanding:194", "bbh:date_understanding:206", "bbh:date_understanding:207", "bbh:date_understanding:222", "bbh:disambiguation_qa:11", "bbh:disambiguation_qa:23", "bbh:disambiguation_qa:24", "bbh:disambiguation_qa:26", "bbh:disambiguation_qa:55", "bbh:disambiguation_qa:67", "bbh:disambiguation_qa:86", "bbh:disambiguation_qa:88", "bbh:disambiguation_qa:91", "bbh:disambiguation_qa:97", "bbh:disambiguation_qa:154", "bbh:disambiguation_qa:195", "bbh:disambiguation_qa:206", "bbh:disambiguation_qa:216", "bbh:disambiguation_qa:245", "bbh:dyck_languages:20", "bbh:dyck_languages:31", "bbh:dyck_languages:75", "bbh:dyck_languages:96", "bbh:dyck_languages:117", "bbh:dyck_languages:137", "bbh:dyck_languages:141", "bbh:dyck_languages:158", "bbh:dyck_languages:160", "bbh:dyck_languages:186", "bbh:dyck_languages:212", "bbh:dyck_languages:220", "bbh:dyck_languages:226", "bbh:dyck_languages:236", "bbh:dyck_languages:249", "bbh:formal_fallacies:11", "bbh:formal_fallacies:17", "bbh:formal_fallacies:20", "bbh:formal_fallacies:25", "bbh:formal_fallacies:49", "bbh:formal_fallacies:58", "bbh:formal_fallacies:59", "bbh:formal_fallacies:74", "bbh:formal_fallacies:92", "bbh:formal_fallacies:147", "bbh:formal_fallacies:169", "bbh:formal_fallacies:180", "bbh:formal_fallacies:197", "bbh:formal_fallacies:218", "bbh:formal_fallacies:221", "bbh:geometric_shapes:41", "bbh:geometric_shapes:53", "bbh:geometric_shapes:68", "bbh:geometric_shapes:71", "bbh:geometric_shapes:90", "bbh:geometric_shapes:93", "bbh:geometric_shapes:94", "bbh:geometric_shapes:97", "bbh:geometric_shapes:116", "bbh:geometric_shapes:162", "bbh:geometric_shapes:171", "bbh:geometric_shapes:174", "bbh:geometric_shapes:179", "bbh:geometric_shapes:213", "bbh:geometric_shapes:239", "bbh:hyperbaton:18", "bbh:hyperbaton:41", "bbh:hyperbaton:43", "bbh:hyperbaton:62", "bbh:hyperbaton:69", "bbh:hyperbaton:97", "bbh:hyperbaton:118", "bbh:hyperbaton:136", "bbh:hyperbaton:155", "bbh:hyperbaton:162", "bbh:hyperbaton:163", "bbh:hyperbaton:165", "bbh:hyperbaton:176", "bbh:hyperbaton:186", "bbh:hyperbaton:236", "bbh:logical_deduction_five_objects:8", "bbh:logical_deduction_five_objects:14", "bbh:logical_deduction_five_objects:56", "bbh:logical_deduction_five_objects:58", "bbh:logical_deduction_five_objects:68", "bbh:logical_deduction_five_objects:80", "bbh:logical_deduction_five_objects:83", "bbh:logical_deduction_five_objects:102", "bbh:logical_deduction_five_objects:142", "bbh:logical_deduction_five_objects:175", "bbh:logical_deduction_five_objects:196", "bbh:logical_deduction_five_objects:198", "bbh:logical_deduction_five_objects:206", "bbh:logical_deduction_five_objects:210", "bbh:logical_deduction_five_objects:215", "bbh:logical_deduction_seven_objects:16", "bbh:logical_deduction_seven_objects:54", "bbh:logical_deduction_seven_objects:80", "bbh:logical_deduction_seven_objects:101", "bbh:logical_deduction_seven_objects:117", "bbh:logical_deduction_seven_objects:127", "bbh:logical_deduction_seven_objects:145", "bbh:logical_deduction_seven_objects:164", "bbh:logical_deduction_seven_objects:167", "bbh:logical_deduction_seven_objects:183", "bbh:logical_deduction_seven_objects:224", "bbh:logical_deduction_seven_objects:226", "bbh:logical_deduction_seven_objects:233", "bbh:logical_deduction_seven_objects:234", "bbh:logical_deduction_seven_objects:241", "bbh:logical_deduction_three_objects:35", "bbh:logical_deduction_three_objects:36", "bbh:logical_deduction_three_objects:56", "bbh:logical_deduction_three_objects:63", "bbh:logical_deduction_three_objects:67", "bbh:logical_deduction_three_objects:92", "bbh:logical_deduction_three_objects:102", "bbh:logical_deduction_three_objects:109", "bbh:logical_deduction_three_objects:130", "bbh:logical_deduction_three_objects:137", "bbh:logical_deduction_three_objects:143", "bbh:logical_deduction_three_objects:149", "bbh:logical_deduction_three_objects:190", "bbh:logical_deduction_three_objects:191", "bbh:logical_deduction_three_objects:229", "bbh:movie_recommendation:12", "bbh:movie_recommendation:16", "bbh:movie_recommendation:23", "bbh:movie_recommendation:28", "bbh:movie_recommendation:39", "bbh:movie_recommendation:40", "bbh:movie_recommendation:98", "bbh:movie_recommendation:108", "bbh:movie_recommendation:126", "bbh:movie_recommendation:152", "bbh:movie_recommendation:160", "bbh:movie_recommendation:174", "bbh:movie_recommendation:193", "bbh:movie_recommendation:202", "bbh:movie_recommendation:220", "bbh:multistep_arithmetic_two:2", "bbh:multistep_arithmetic_two:29", "bbh:multistep_arithmetic_two:64", "bbh:multistep_arithmetic_two:97", "bbh:multistep_arithmetic_two:119", "bbh:multistep_arithmetic_two:135", "bbh:multistep_arithmetic_two:137", "bbh:multistep_arithmetic_two:141", "bbh:multistep_arithmetic_two:152", "bbh:multistep_arithmetic_two:174", "bbh:multistep_arithmetic_two:184", "bbh:multistep_arithmetic_two:220", "bbh:multistep_arithmetic_two:226", "bbh:multistep_arithmetic_two:241", "bbh:multistep_arithmetic_two:248", "bbh:navigate:0", "bbh:navigate:28", "bbh:navigate:40", "bbh:navigate:67", "bbh:navigate:68", "bbh:navigate:75", "bbh:navigate:87", "bbh:navigate:111", "bbh:navigate:116", "bbh:navigate:164", "bbh:navigate:184", "bbh:navigate:192", "bbh:navigate:196", "bbh:navigate:224", "bbh:navigate:244", "bbh:object_counting:27", "bbh:object_counting:39", "bbh:object_counting:45", "bbh:object_counting:50", "bbh:object_counting:76", "bbh:object_counting:128", "bbh:object_counting:129", "bbh:object_counting:155", "bbh:object_counting:160", "bbh:object_counting:163", "bbh:object_counting:195", "bbh:object_counting:215", "bbh:object_counting:222", "bbh:object_counting:233", "bbh:object_counting:248", "bbh:penguins_in_a_table:0", "bbh:penguins_in_a_table:4", "bbh:penguins_in_a_table:14", "bbh:penguins_in_a_table:20", "bbh:penguins_in_a_table:28", "bbh:penguins_in_a_table:41", "bbh:penguins_in_a_table:61", "bbh:penguins_in_a_table:78", "bbh:penguins_in_a_table:82", "bbh:penguins_in_a_table:92", "bbh:penguins_in_a_table:95", "bbh:penguins_in_a_table:125", "bbh:penguins_in_a_table:135", "bbh:penguins_in_a_table:138", "bbh:penguins_in_a_table:145", "bbh:reasoning_about_colored_objects:17", "bbh:reasoning_about_colored_objects:21", "bbh:reasoning_about_colored_objects:32", "bbh:reasoning_about_colored_objects:42", "bbh:reasoning_about_colored_objects:67", "bbh:reasoning_about_colored_objects:121", "bbh:reasoning_about_colored_objects:124", "bbh:reasoning_about_colored_objects:136", "bbh:reasoning_about_colored_objects:140", "bbh:reasoning_about_colored_objects:168", "bbh:reasoning_about_colored_objects:187", "bbh:reasoning_about_colored_objects:194", "bbh:reasoning_about_colored_objects:196", "bbh:reasoning_about_colored_objects:208", "bbh:reasoning_about_colored_objects:242", "bbh:ruin_names:51", "bbh:ruin_names:54", "bbh:ruin_names:79", "bbh:ruin_names:102", "bbh:ruin_names:108", "bbh:ruin_names:135", "bbh:ruin_names:138", "bbh:ruin_names:155", "bbh:ruin_names:176", "bbh:ruin_names:182", "bbh:ruin_names:186", "bbh:ruin_names:193", "bbh:ruin_names:223", "bbh:ruin_names:237", "bbh:ruin_names:246", "bbh:salient_translation_error_detection:5", "bbh:salient_translation_error_detection:16", "bbh:salient_translation_error_detection:30", "bbh:salient_translation_error_detection:57", "bbh:salient_translation_error_detection:63", "bbh:salient_translation_error_detection:86", "bbh:salient_translation_error_detection:95", "bbh:salient_translation_error_detection:112", "bbh:salient_translation_error_detection:115", "bbh:salient_translation_error_detection:132", "bbh:salient_translation_error_detection:141", "bbh:salient_translation_error_detection:150", "bbh:salient_translation_error_detection:166", "bbh:salient_translation_error_detection:171", "bbh:salient_translation_error_detection:230", "bbh:snarks:1", "bbh:snarks:8", "bbh:snarks:15", "bbh:snarks:17", "bbh:snarks:18", "bbh:snarks:56", "bbh:snarks:58", "bbh:snarks:60", "bbh:snarks:71", "bbh:snarks:84", "bbh:snarks:124", "bbh:snarks:131", "bbh:snarks:150", "bbh:snarks:161", "bbh:snarks:171", "bbh:sports_understanding:24", "bbh:sports_understanding:33", "bbh:sports_understanding:48", "bbh:sports_understanding:54", "bbh:sports_understanding:62", "bbh:sports_understanding:104", "bbh:sports_understanding:121", "bbh:sports_understanding:138", "bbh:sports_understanding:146", "bbh:sports_understanding:147", "bbh:sports_understanding:185", "bbh:sports_understanding:200", "bbh:sports_understanding:206", "bbh:sports_understanding:225", "bbh:sports_understanding:239", "bbh:temporal_sequences:13", "bbh:temporal_sequences:15", "bbh:temporal_sequences:24", "bbh:temporal_sequences:25", "bbh:temporal_sequences:90", "bbh:temporal_sequences:105", "bbh:temporal_sequences:108", "bbh:temporal_sequences:110", "bbh:temporal_sequences:119", "bbh:temporal_sequences:165", "bbh:temporal_sequences:167", "bbh:temporal_sequences:168", "bbh:temporal_sequences:172", "bbh:temporal_sequences:186", "bbh:temporal_sequences:221", "bbh:tracking_shuffled_objects_five_objects:27", "bbh:tracking_shuffled_objects_five_objects:35", "bbh:tracking_shuffled_objects_five_objects:46", "bbh:tracking_shuffled_objects_five_objects:48", "bbh:tracking_shuffled_objects_five_objects:49", "bbh:tracking_shuffled_objects_five_objects:63", "bbh:tracking_shuffled_objects_five_objects:71", "bbh:tracking_shuffled_objects_five_objects:86", "bbh:tracking_shuffled_objects_five_objects:103", "bbh:tracking_shuffled_objects_five_objects:108", "bbh:tracking_shuffled_objects_five_objects:114", "bbh:tracking_shuffled_objects_five_objects:137", "bbh:tracking_shuffled_objects_five_objects:186", "bbh:tracking_shuffled_objects_five_objects:204", "bbh:tracking_shuffled_objects_five_objects:220", "bbh:tracking_shuffled_objects_seven_objects:12", "bbh:tracking_shuffled_objects_seven_objects:19", "bbh:tracking_shuffled_objects_seven_objects:25", "bbh:tracking_shuffled_objects_seven_objects:63", "bbh:tracking_shuffled_objects_seven_objects:113", "bbh:tracking_shuffled_objects_seven_objects:118", "bbh:tracking_shuffled_objects_seven_objects:138", "bbh:tracking_shuffled_objects_seven_objects:140", "bbh:tracking_shuffled_objects_seven_objects:166", "bbh:tracking_shuffled_objects_seven_objects:206", "bbh:tracking_shuffled_objects_seven_objects:214", "bbh:tracking_shuffled_objects_seven_objects:219", "bbh:tracking_shuffled_objects_seven_objects:220", "bbh:tracking_shuffled_objects_seven_objects:223", "bbh:tracking_shuffled_objects_seven_objects:236", "bbh:tracking_shuffled_objects_three_objects:3", "bbh:tracking_shuffled_objects_three_objects:23", "bbh:tracking_shuffled_objects_three_objects:42", "bbh:tracking_shuffled_objects_three_objects:54", "bbh:tracking_shuffled_objects_three_objects:60", "bbh:tracking_shuffled_objects_three_objects:102", "bbh:tracking_shuffled_objects_three_objects:104", "bbh:tracking_shuffled_objects_three_objects:123", "bbh:tracking_shuffled_objects_three_objects:124", "bbh:tracking_shuffled_objects_three_objects:192", "bbh:tracking_shuffled_objects_three_objects:217", "bbh:tracking_shuffled_objects_three_objects:221", "bbh:tracking_shuffled_objects_three_objects:231", "bbh:tracking_shuffled_objects_three_objects:237", "bbh:tracking_shuffled_objects_three_objects:248", "bbh:web_of_lies:0", "bbh:web_of_lies:15", "bbh:web_of_lies:42", "bbh:web_of_lies:67", "bbh:web_of_lies:73", "bbh:web_of_lies:97", "bbh:web_of_lies:99", "bbh:web_of_lies:108", "bbh:web_of_lies:116", "bbh:web_of_lies:142", "bbh:web_of_lies:178", "bbh:web_of_lies:187", "bbh:web_of_lies:200", "bbh:web_of_lies:237", "bbh:web_of_lies:245", "bbh:word_sorting:14", "bbh:word_sorting:15", "bbh:word_sorting:39", "bbh:word_sorting:48", "bbh:word_sorting:55", "bbh:word_sorting:75", "bbh:word_sorting:80", "bbh:word_sorting:124", "bbh:word_sorting:138", "bbh:word_sorting:148", "bbh:word_sorting:169", "bbh:word_sorting:183", "bbh:word_sorting:188", "bbh:word_sorting:191", "bbh:word_sorting:247" ] }, "mmlupro": { "source_urls": [ "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro" ], "count": 350, "task_ids": [ "mmlupro:2846", "mmlupro:2857", "mmlupro:2864", "mmlupro:2875", "mmlupro:2876", "mmlupro:2888", "mmlupro:2889", "mmlupro:2928", "mmlupro:2967", "mmlupro:2996", "mmlupro:3046", "mmlupro:3058", "mmlupro:3222", "mmlupro:3238", "mmlupro:3297", "mmlupro:3323", "mmlupro:3329", "mmlupro:3352", "mmlupro:3392", "mmlupro:3401", "mmlupro:3407", "mmlupro:3417", "mmlupro:3418", "mmlupro:3443", "mmlupro:3500", "mmlupro:79", "mmlupro:145", "mmlupro:173", "mmlupro:205", "mmlupro:281", "mmlupro:316", "mmlupro:339", "mmlupro:343", "mmlupro:380", "mmlupro:394", "mmlupro:396", "mmlupro:479", "mmlupro:543", "mmlupro:544", "mmlupro:610", "mmlupro:652", "mmlupro:654", "mmlupro:673", "mmlupro:713", "mmlupro:737", "mmlupro:750", "mmlupro:762", "mmlupro:764", "mmlupro:810", "mmlupro:846", "mmlupro:3542", "mmlupro:3666", "mmlupro:3676", "mmlupro:3740", "mmlupro:3747", "mmlupro:3764", "mmlupro:3799", "mmlupro:3803", "mmlupro:3846", "mmlupro:3851", "mmlupro:3964", "mmlupro:4030", "mmlupro:4071", "mmlupro:4073", "mmlupro:4088", "mmlupro:4114", "mmlupro:4144", "mmlupro:4150", "mmlupro:4246", "mmlupro:4288", "mmlupro:4432", "mmlupro:4573", "mmlupro:4620", "mmlupro:4638", "mmlupro:4649", "mmlupro:10357", "mmlupro:10378", "mmlupro:10383", "mmlupro:10405", "mmlupro:10425", "mmlupro:10441", "mmlupro:10463", "mmlupro:10466", "mmlupro:10487", "mmlupro:10493", "mmlupro:10494", "mmlupro:10500", "mmlupro:10503", "mmlupro:10530", "mmlupro:10535", "mmlupro:10576", "mmlupro:10612", "mmlupro:10620", "mmlupro:10672", "mmlupro:10687", "mmlupro:10689", "mmlupro:10715", "mmlupro:10731", "mmlupro:10743", "mmlupro:10758", "mmlupro:6836", "mmlupro:6863", "mmlupro:6867", "mmlupro:6869", "mmlupro:6904", "mmlupro:6941", "mmlupro:6957", "mmlupro:6978", "mmlupro:6979", "mmlupro:7044", "mmlupro:7146", "mmlupro:7198", "mmlupro:7205", "mmlupro:7210", "mmlupro:7271", "mmlupro:7274", "mmlupro:7286", "mmlupro:7398", "mmlupro:7404", "mmlupro:7405", "mmlupro:7414", "mmlupro:7436", "mmlupro:7550", "mmlupro:7565", "mmlupro:7658", "mmlupro:11310", "mmlupro:11391", "mmlupro:11444", "mmlupro:11452", "mmlupro:11467", "mmlupro:11469", "mmlupro:11529", "mmlupro:11542", "mmlupro:11649", "mmlupro:11703", "mmlupro:11709", "mmlupro:11860", "mmlupro:11923", "mmlupro:11970", "mmlupro:11986", "mmlupro:12055", "mmlupro:12086", "mmlupro:12107", "mmlupro:12118", "mmlupro:12173", "mmlupro:12183", "mmlupro:12190", "mmlupro:12193", "mmlupro:12235", "mmlupro:12240", "mmlupro:6026", "mmlupro:6041", "mmlupro:6113", "mmlupro:6166", "mmlupro:6208", "mmlupro:6231", "mmlupro:6232", "mmlupro:6237", "mmlupro:6258", "mmlupro:6277", "mmlupro:6317", "mmlupro:6345", "mmlupro:6363", "mmlupro:6397", "mmlupro:6427", "mmlupro:6477", "mmlupro:6487", "mmlupro:6682", "mmlupro:6692", "mmlupro:6726", "mmlupro:6760", "mmlupro:6762", "mmlupro:6809", "mmlupro:6814", "mmlupro:6822", "mmlupro:4683", "mmlupro:4688", "mmlupro:4704", "mmlupro:4724", "mmlupro:4728", "mmlupro:4762", "mmlupro:4771", "mmlupro:4808", "mmlupro:4810", "mmlupro:4817", "mmlupro:4836", "mmlupro:4844", "mmlupro:4845", "mmlupro:4852", "mmlupro:4855", "mmlupro:4880", "mmlupro:4898", "mmlupro:4899", "mmlupro:4936", "mmlupro:4950", "mmlupro:4973", "mmlupro:4981", "mmlupro:5004", "mmlupro:5024", "mmlupro:5051", "mmlupro:869", "mmlupro:956", "mmlupro:1011", "mmlupro:1106", "mmlupro:1125", "mmlupro:1130", "mmlupro:1262", "mmlupro:1265", "mmlupro:1276", "mmlupro:1397", "mmlupro:1485", "mmlupro:1495", "mmlupro:1513", "mmlupro:1522", "mmlupro:1548", "mmlupro:1556", "mmlupro:1626", "mmlupro:1670", "mmlupro:1707", "mmlupro:1719", "mmlupro:1767", "mmlupro:1777", "mmlupro:1923", "mmlupro:1932", "mmlupro:1949", "mmlupro:7687", "mmlupro:8034", "mmlupro:8044", "mmlupro:8118", "mmlupro:8125", "mmlupro:8277", "mmlupro:8306", "mmlupro:8312", "mmlupro:8350", "mmlupro:8467", "mmlupro:8522", "mmlupro:8552", "mmlupro:8571", "mmlupro:8595", "mmlupro:8596", "mmlupro:8643", "mmlupro:8660", "mmlupro:8738", "mmlupro:8814", "mmlupro:8857", "mmlupro:8879", "mmlupro:8935", "mmlupro:8953", "mmlupro:9033", "mmlupro:9042", "mmlupro:5085", "mmlupro:5107", "mmlupro:5134", "mmlupro:5146", "mmlupro:5155", "mmlupro:5210", "mmlupro:5263", "mmlupro:5290", "mmlupro:5300", "mmlupro:5310", "mmlupro:5350", "mmlupro:5378", "mmlupro:5405", "mmlupro:5552", "mmlupro:5594", "mmlupro:5697", "mmlupro:5706", "mmlupro:5720", "mmlupro:5752", "mmlupro:5761", "mmlupro:5845", "mmlupro:5862", "mmlupro:5901", "mmlupro:5914", "mmlupro:5947", "mmlupro:10776", "mmlupro:10831", "mmlupro:10852", "mmlupro:10867", "mmlupro:10876", "mmlupro:10889", "mmlupro:10901", "mmlupro:10974", "mmlupro:10983", "mmlupro:10991", "mmlupro:10996", "mmlupro:11013", "mmlupro:11034", "mmlupro:11075", "mmlupro:11104", "mmlupro:11119", "mmlupro:11137", "mmlupro:11141", "mmlupro:11152", "mmlupro:11169", "mmlupro:11179", "mmlupro:11183", "mmlupro:11225", "mmlupro:11238", "mmlupro:11242", "mmlupro:9146", "mmlupro:9292", "mmlupro:9317", "mmlupro:9369", "mmlupro:9550", "mmlupro:9554", "mmlupro:9574", "mmlupro:9611", "mmlupro:9694", "mmlupro:9920", "mmlupro:9954", "mmlupro:9962", "mmlupro:9970", "mmlupro:9983", "mmlupro:10000", "mmlupro:10021", "mmlupro:10083", "mmlupro:10112", "mmlupro:10120", "mmlupro:10140", "mmlupro:10176", "mmlupro:10196", "mmlupro:10199", "mmlupro:10275", "mmlupro:10311", "mmlupro:2054", "mmlupro:2068", "mmlupro:2071", "mmlupro:2131", "mmlupro:2144", "mmlupro:2146", "mmlupro:2209", "mmlupro:2226", "mmlupro:2230", "mmlupro:2234", "mmlupro:2269", "mmlupro:2272", "mmlupro:2283", "mmlupro:2318", "mmlupro:2334", "mmlupro:2383", "mmlupro:2409", "mmlupro:2416", "mmlupro:2443", "mmlupro:2489", "mmlupro:2546", "mmlupro:2638", "mmlupro:2711", "mmlupro:2725", "mmlupro:2732" ] }, "belebele_cs": { "source_urls": [ "https://huggingface.co/datasets/facebook/belebele" ], "count": 200, "task_ids": [ "belebele_cs:6", "belebele_cs:10", "belebele_cs:12", "belebele_cs:19", "belebele_cs:20", "belebele_cs:26", "belebele_cs:27", "belebele_cs:29", "belebele_cs:31", "belebele_cs:41", "belebele_cs:42", "belebele_cs:51", "belebele_cs:53", "belebele_cs:63", "belebele_cs:70", "belebele_cs:77", "belebele_cs:83", "belebele_cs:85", "belebele_cs:87", "belebele_cs:88", "belebele_cs:104", "belebele_cs:108", "belebele_cs:110", "belebele_cs:117", "belebele_cs:123", "belebele_cs:129", "belebele_cs:130", "belebele_cs:138", "belebele_cs:156", "belebele_cs:167", "belebele_cs:169", "belebele_cs:171", "belebele_cs:180", "belebele_cs:186", "belebele_cs:195", "belebele_cs:203", "belebele_cs:204", "belebele_cs:211", "belebele_cs:216", "belebele_cs:217", "belebele_cs:223", "belebele_cs:224", "belebele_cs:225", "belebele_cs:229", "belebele_cs:244", "belebele_cs:248", "belebele_cs:249", "belebele_cs:251", "belebele_cs:253", "belebele_cs:258", "belebele_cs:262", "belebele_cs:264", "belebele_cs:266", "belebele_cs:274", "belebele_cs:276", "belebele_cs:279", "belebele_cs:282", "belebele_cs:284", "belebele_cs:285", "belebele_cs:286", "belebele_cs:305", "belebele_cs:308", "belebele_cs:318", "belebele_cs:319", "belebele_cs:330", "belebele_cs:332", "belebele_cs:334", "belebele_cs:335", "belebele_cs:338", "belebele_cs:344", "belebele_cs:345", "belebele_cs:348", "belebele_cs:349", "belebele_cs:355", "belebele_cs:358", "belebele_cs:360", "belebele_cs:363", "belebele_cs:370", "belebele_cs:377", "belebele_cs:379", "belebele_cs:384", "belebele_cs:388", "belebele_cs:389", "belebele_cs:398", "belebele_cs:399", "belebele_cs:403", "belebele_cs:406", "belebele_cs:414", "belebele_cs:425", "belebele_cs:429", "belebele_cs:430", "belebele_cs:431", "belebele_cs:438", "belebele_cs:443", "belebele_cs:445", "belebele_cs:446", "belebele_cs:461", "belebele_cs:465", "belebele_cs:470", "belebele_cs:472", "belebele_cs:475", "belebele_cs:476", "belebele_cs:477", "belebele_cs:478", "belebele_cs:481", "belebele_cs:484", "belebele_cs:488", "belebele_cs:495", "belebele_cs:497", "belebele_cs:499", "belebele_cs:501", "belebele_cs:506", "belebele_cs:518", "belebele_cs:529", "belebele_cs:534", "belebele_cs:545", "belebele_cs:546", "belebele_cs:551", "belebele_cs:552", "belebele_cs:555", "belebele_cs:559", "belebele_cs:569", "belebele_cs:577", "belebele_cs:579", "belebele_cs:583", "belebele_cs:589", "belebele_cs:592", "belebele_cs:598", "belebele_cs:606", "belebele_cs:610", "belebele_cs:614", "belebele_cs:617", "belebele_cs:620", "belebele_cs:621", "belebele_cs:624", "belebele_cs:630", "belebele_cs:636", "belebele_cs:637", "belebele_cs:642", "belebele_cs:650", "belebele_cs:658", "belebele_cs:663", "belebele_cs:665", "belebele_cs:666", "belebele_cs:667", "belebele_cs:677", "belebele_cs:678", "belebele_cs:684", "belebele_cs:685", "belebele_cs:693", "belebele_cs:695", "belebele_cs:700", "belebele_cs:707", "belebele_cs:712", "belebele_cs:713", "belebele_cs:716", "belebele_cs:724", "belebele_cs:732", "belebele_cs:735", "belebele_cs:741", "belebele_cs:752", "belebele_cs:765", "belebele_cs:767", "belebele_cs:770", "belebele_cs:771", "belebele_cs:772", "belebele_cs:779", "belebele_cs:783", "belebele_cs:785", "belebele_cs:788", "belebele_cs:795", "belebele_cs:798", "belebele_cs:799", "belebele_cs:803", "belebele_cs:813", "belebele_cs:817", "belebele_cs:819", "belebele_cs:827", "belebele_cs:834", "belebele_cs:837", "belebele_cs:840", "belebele_cs:845", "belebele_cs:846", "belebele_cs:848", "belebele_cs:852", "belebele_cs:855", "belebele_cs:858", "belebele_cs:859", "belebele_cs:861", "belebele_cs:863", "belebele_cs:866", "belebele_cs:872", "belebele_cs:873", "belebele_cs:876", "belebele_cs:877", "belebele_cs:878", "belebele_cs:884", "belebele_cs:887", "belebele_cs:891", "belebele_cs:892" ] }, "cz_agree": { "source_urls": [ "https://huggingface.co/datasets/davidadamczyk/czechbench_agree" ], "count": 150, "task_ids": [ "cz_agree:1", "cz_agree:12", "cz_agree:14", "cz_agree:16", "cz_agree:17", "cz_agree:30", "cz_agree:36", "cz_agree:38", "cz_agree:46", "cz_agree:52", "cz_agree:54", "cz_agree:55", "cz_agree:58", "cz_agree:61", "cz_agree:63", "cz_agree:66", "cz_agree:72", "cz_agree:75", "cz_agree:76", "cz_agree:80", "cz_agree:83", "cz_agree:84", "cz_agree:87", "cz_agree:88", "cz_agree:95", "cz_agree:101", "cz_agree:102", "cz_agree:105", "cz_agree:115", "cz_agree:116", "cz_agree:118", "cz_agree:119", "cz_agree:121", "cz_agree:122", "cz_agree:123", "cz_agree:129", "cz_agree:135", "cz_agree:136", "cz_agree:144", "cz_agree:147", "cz_agree:149", "cz_agree:152", "cz_agree:155", "cz_agree:157", "cz_agree:160", "cz_agree:183", "cz_agree:188", "cz_agree:189", "cz_agree:196", "cz_agree:197", "cz_agree:198", "cz_agree:209", "cz_agree:212", "cz_agree:216", "cz_agree:217", "cz_agree:221", "cz_agree:224", "cz_agree:226", "cz_agree:230", "cz_agree:231", "cz_agree:232", "cz_agree:236", "cz_agree:240", "cz_agree:245", "cz_agree:246", "cz_agree:247", "cz_agree:249", "cz_agree:254", "cz_agree:257", "cz_agree:258", "cz_agree:270", "cz_agree:277", "cz_agree:278", "cz_agree:280", "cz_agree:282", "cz_agree:283", "cz_agree:289", "cz_agree:291", "cz_agree:299", "cz_agree:301", "cz_agree:303", "cz_agree:304", "cz_agree:309", "cz_agree:311", "cz_agree:312", "cz_agree:313", "cz_agree:320", "cz_agree:323", "cz_agree:331", "cz_agree:338", "cz_agree:348", "cz_agree:351", "cz_agree:352", "cz_agree:359", "cz_agree:360", "cz_agree:361", "cz_agree:369", "cz_agree:377", "cz_agree:383", "cz_agree:389", "cz_agree:403", "cz_agree:405", "cz_agree:410", "cz_agree:411", "cz_agree:414", "cz_agree:416", "cz_agree:422", "cz_agree:423", "cz_agree:426", "cz_agree:435", "cz_agree:439", "cz_agree:440", "cz_agree:451", "cz_agree:453", "cz_agree:458", "cz_agree:461", "cz_agree:464", "cz_agree:465", "cz_agree:473", "cz_agree:478", "cz_agree:480", "cz_agree:484", "cz_agree:485", "cz_agree:487", "cz_agree:489", "cz_agree:490", "cz_agree:491", "cz_agree:495", "cz_agree:497", "cz_agree:500", "cz_agree:502", "cz_agree:503", "cz_agree:517", "cz_agree:521", "cz_agree:537", "cz_agree:543", "cz_agree:546", "cz_agree:547", "cz_agree:548", "cz_agree:550", "cz_agree:556", "cz_agree:562", "cz_agree:563", "cz_agree:566", "cz_agree:568", "cz_agree:584", "cz_agree:587", "cz_agree:592", "cz_agree:596", "cz_agree:600" ] }, "drop": { "source_urls": [ "https://huggingface.co/datasets/ucinlp/drop" ], "count": 200, "task_ids": [ "drop:8b442d27-241b-4fd9-87d4-1e5bee19cb2e", "drop:e9636c00-bafb-4bc4-a5a3-1be4104e6c8f", "drop:7763f67a-9aa1-4c2b-9ea8-0b7c8f95fb7a", "drop:1b005c22-8b1d-4c06-95a0-97a7ccb94652", "drop:89fe8219-b100-4200-beff-e803747f2eb4", "drop:941dc59f-89f1-45e2-96be-19b34145fbfe", "drop:ea113b9d-f2ee-4162-b8a3-5450452f06ff", "drop:4dbbe943-0043-4196-b26c-0c523c2266f4", "drop:e824217f-0ec6-4b50-bf08-a6f06a0d498c", "drop:a6814037-1542-4e97-86df-23f7723fe41e", "drop:9f9d36cf-9de2-4d14-b27d-b6eea20f603c", "drop:b121656d-0bc4-4532-bf34-453a83d84f84", "drop:d4d36574-7e49-4ec8-86c1-726edab89951", "drop:b6a01cec-3389-46eb-b926-4ef75b332f13", "drop:ee329d5c-63f6-4b76-9245-416b9539bb67", "drop:53397865-5cfe-4e5f-ab3a-7cc45baf1c5e", "drop:dc48b619-5890-4ce1-9820-7fd2adab4388", "drop:d50fe7c6-b603-402e-9999-06a33228126e", "drop:75d5a26b-5736-4d2a-9489-e632b7cbf868", "drop:eeacc39f-b238-4dfd-8765-424be50a4cdc", "drop:7fc77120-c308-4204-9b11-6487589aa7df", "drop:6391c4b5-a47a-415f-b982-89bb1acd0e21", "drop:2aa85a31-c18f-4b5e-a5de-a6ee4558a50b", "drop:9ed332d1-8f75-422d-9e98-c3ce311d9996", "drop:f6cd00e5-2eda-46fd-9d2b-7141ed39afa0", "drop:1abf3266-095d-424b-9d2d-f4ea42222be4", "drop:b6b2a97f-f496-454f-bfb6-376847bf151c", "drop:0420a4f4-fc96-4436-a6c7-fc799f0f6acd", "drop:9b2cfd65-8042-4085-9d3e-876428cb5158", "drop:c835a0f4-07b5-4362-b4e6-b9ba6c9b981b", "drop:6b1bd1f5-3053-4a71-a93f-1b9d38b7e54a", "drop:f648c784-6d09-4146-b21b-1e276656bdf4", "drop:e8b67d25-2c18-43c0-8a72-34d85170b4e0", "drop:cb86e001-b8c8-4534-bcc3-b25b34ce2268", "drop:7095dfb6-88a4-4dee-9dc3-f0efa7332672", "drop:e116a51c-e22f-4f42-a37c-2019ae7d6bd6", "drop:1628225b-efe7-4282-8066-6a26eb03bfaf", "drop:7da66062-7794-401b-ab4c-a76b68193ddc", "drop:9556e1aa-cf1e-49f2-97e3-08a4b40eb769", "drop:9b6bd0fc-f8db-453b-a190-046c6cd6ff26", "drop:43c7e507-7f10-4341-b239-fbf7d1a2f1d1", "drop:46ed7588-8ae6-4617-a033-3affe597dd4b", "drop:85824081-8228-451e-bdfd-ccf7f2845be7", "drop:c5482979-e6ef-406a-b3e1-eda45e0eac29", "drop:5f6238e0-1d79-472b-9eaa-9f76e8714100", "drop:a588aaaf-070a-4704-8a7d-11e03cc3ada1", "drop:1882d944-04bb-4e85-8c7e-3b71a8bf0f28", "drop:76194595-2ca3-437a-a6c3-8ee1a6092da5", "drop:cdec287d-5873-44d4-ae17-c1d2e53e9830", "drop:98d66709-4510-4649-a268-2fe455c54a32", "drop:6f66d4a7-7fd3-465c-97a3-819333307196", "drop:75d9e7ff-ba9a-441d-a271-09234f905e05", "drop:93a15ec2-dddf-4fd9-b630-b831b6b1a89a", "drop:13dfc499-a468-44fe-895f-f59f5527c47d", "drop:f17afe43-56d0-4ce1-9ed6-6da94f155951", "drop:500a948d-6e09-46a1-b675-7f2aeb583bb3", "drop:1ab20dd5-3cc9-4345-a0b1-629d19e1c6f2", "drop:3cee3cf2-f925-491d-8b6e-46dc74c26223", "drop:dbb0ac7b-7624-4d99-90b8-bcb7454877b7", "drop:5351d216-8f79-4a0f-b7b3-fd4053a34d5b", "drop:7f995e5b-d270-4293-b3f9-bfb499b1a1da", "drop:a159f705-e3e5-4cc4-bb05-b5f50ea140e6", "drop:631e663c-e183-41dc-9700-91103f2ca556", "drop:867bae19-bab7-4f9e-af8a-5359b7fa17fd", "drop:054d0277-77da-461b-974d-594d4b6bcae1", "drop:8d5b2fe8-a0a2-4a51-afd5-5b7154babbf8", "drop:9bf015d2-429d-4b2a-97dc-1b861a5c2e97", "drop:206640e8-4573-4a4d-9533-8935ea2e2011", "drop:68d02b1b-285b-4e37-83b0-34b3d403c976", "drop:d1b80ed3-5eea-4397-9c19-3c4606e36a84", "drop:46f4cfa2-1ee7-4640-9d9e-d60a57605eab", "drop:f0b18ae3-a7ed-427b-89a8-e493fa8b6391", "drop:4cbc5ffa-03d8-4c17-b745-1d5e62265b96", "drop:cd03169b-be10-4321-bd9b-10de3954fe3a", "drop:72b883ac-acdd-4bc7-a182-ebdf76c0fe6e", "drop:e1c0558d-fe69-4a29-abeb-6f723ea243e9", "drop:d238552b-0bcb-4483-b72d-791b8b093cef", "drop:9283a2b5-ab15-4b28-9ca6-2d2c648a3bbf", "drop:ff9c400d-a718-4af3-a8b3-e5115e561f25", "drop:ebafc5a1-b754-45ab-9a1e-3c01cddbfe58", "drop:dee73326-75da-4136-ad22-e0451a652607", "drop:1aed8cf7-96a2-4c0d-af61-ff771944db22", "drop:65683a61-f053-4a15-b385-077479a7329e", "drop:c90aa9e6-8ee3-4827-9f54-18367ac73ace", "drop:1a0d7a7a-bbe1-476e-9679-3f94af700f9c", "drop:1e81fe2c-cb65-40a1-85e1-cf6e57cb25a2", "drop:1fd98d0e-bd8a-494a-8d1c-d48a98504b30", "drop:5fc913ff-be3f-491b-a026-fb71a63e3483", "drop:ba04b5c7-b0ea-45c4-81c6-e32db9d323d8", "drop:ef89950e-6ae3-46cb-8f69-c42e68ea6c00", "drop:7800827a-de8c-4e3a-9aae-5790f60791a2", "drop:9fcc4a4f-0c2f-48e0-a506-e5499bc0cd48", "drop:bb37ffa4-1f7c-4fb9-a5bd-b2ee54b8c12a", "drop:6ebee2d1-4f22-4d84-aa07-5bc7285f7c6d", "drop:202dbf6a-f10c-4d03-808c-a4cd25c1cfdf", "drop:01f893f7-3103-4fc2-b989-5afabc55ddbb", "drop:993e7c0f-1507-4f25-adb8-d6db9b64dc2c", "drop:0323c3e8-0b60-4800-a8f4-cd209c60c114", "drop:c419c365-d662-4bdb-94ad-8eda27198d3b", "drop:26781e41-8b47-435b-bba9-748551f12b64", "drop:666a4d2c-75af-49ab-b254-9cf6b3eb0f70", "drop:e75d187c-21c8-4163-9caf-a06dde32aed3", "drop:3d24c687-37d3-4e38-bfca-6fb785f963de", "drop:5a66c284-ecf7-4fca-a34c-ad20aaa1d518", "drop:b759f0ad-e103-4173-a5e2-2d68c586dbf5", "drop:b21eddaf-7488-4dbb-b1f9-9aa1ad8952e5", "drop:23057dbd-a603-4154-9972-f3da6d1704f7", "drop:7939f4a5-9e4e-4d3c-8f6f-51deb26d5471", "drop:202312b0-8beb-4ffa-a737-40a8ce917dd6", "drop:767c9790-8396-4be6-9e8e-883af31d4659", "drop:34f6d191-a2ce-4609-91f8-48e6ff9d3584", "drop:54c460c3-200c-4a9b-8d62-3f67a99d428f", "drop:4fc33069-18bd-460f-8bff-bdf82c0d8fb3", "drop:8f875905-30ea-4d6d-984c-7a5d97e4a5a3", "drop:0ad88258-9b24-4733-bb73-1e32d639bec6", "drop:da91e17e-c26b-4c33-acc1-525e7b5a83d3", "drop:7f1dbd60-46de-49b0-b7a8-000bf6c66b83", "drop:42463651-77d7-4645-9db7-79969a3ede3a", "drop:729bc6f3-8d4e-416b-b748-20e9247ac051", "drop:414253a7-15cf-44aa-b205-045c39299d95", "drop:da855b91-32a1-4511-94be-17d473edf712", "drop:03509398-9311-4956-8b86-228e86b92058", "drop:a7e9bf8d-b442-4851-bc9c-2d1edd3c4075", "drop:4880645d-e22e-4250-a118-4943cb1a4176", "drop:68b2cd13-5bfc-4b76-9f44-8e89dc803ee6", "drop:d1042263-cdd3-4975-abc3-e2425435ef9f", "drop:768301bb-36bc-4827-8e16-fec6304bdece", "drop:5b4728f3-c188-4cdd-b3a4-6890a6c616e9", "drop:d35c7047-0a7a-4a2d-9a12-81f4e4909f91", "drop:f79aa0fb-2c48-4e9d-ae8a-985a28ebf63b", "drop:02451b6e-cfaf-486d-bc8e-e526d9b5d56f", "drop:be526f40-9aef-4e30-9d4d-56a7408b3ae9", "drop:4c6ef4d7-34fd-4aa2-b83c-5f494af6149f", "drop:e176be1d-7725-45d0-8b3f-be112bc7d968", "drop:f813f690-e9c4-42e0-bcd5-957b98f7a563", "drop:8dbd00af-12ef-4636-8e39-c4bc532245b3", "drop:ea6b4921-705c-48bf-b6c0-f5c2b247c70d", "drop:417c1342-9940-4518-a81a-8ca3c07d809d", "drop:9a2ae3fc-1ae8-4645-babd-efff4f03a189", "drop:c8b89174-7660-4c4a-95fa-9762146954e6", "drop:f453bd6d-ed33-4221-8e11-a6707bfa5691", "drop:1b4d7de3-ce2e-4ef4-a1e8-a37437810c53", "drop:5e2185df-dd93-4d03-aea1-799f08ff44cd", "drop:901526df-cb0f-42eb-84e2-ad2f82cac029", "drop:1950b2e2-801c-4c06-9cba-25be7ca5eab4", "drop:f1db2be3-564e-413a-80f5-bcf78993aaeb", "drop:92b270cf-7be1-4d7a-b6c0-ea568229b5a5", "drop:755e4e34-ee65-4748-b5a1-f47657d62a31", "drop:3fc91a8a-5db3-4538-97b7-028352c4259f", "drop:01086afb-5f5e-47a7-8c90-fa88a664c270", "drop:0890b283-e99f-4e72-9f01-22e4e2ab6cce", "drop:7e46a226-4902-4918-8cb6-1073f3febafa", "drop:c6610c4f-f793-49f8-b8fd-1ba168c83050", "drop:aebd0e9f-58a0-4003-aebb-5568cf2997eb", "drop:bd4c2c96-9713-4a98-b76d-10eb147699b3", "drop:ab6399ed-91d4-4163-a39d-90850ddec9cc", "drop:9f3152f8-4037-480b-bf3c-8b6ceaf83079", "drop:5094fb87-a173-4cec-b78a-8467b7fdcdc6", "drop:98f8ed5f-7505-4f7f-9fcf-5d627d6a0e1b", "drop:c1bb7b86-5835-4cd1-95d3-73c86cdadd1c", "drop:e6ddabe4-a5ba-404e-9e7b-fd0584be12ea", "drop:2a550127-6805-4c6f-9e41-a36180e44618", "drop:d6d7513a-a473-4dce-b3ae-34713197c98c", "drop:d51e3126-9765-40cf-92a8-409203cf05e5", "drop:9e081dcd-b8dc-469e-9f21-b7250061dd1e", "drop:93944496-8d4b-49d5-8c78-a4f1ac8c544a", "drop:9273e19c-688e-407d-99a8-4b98cd443460", "drop:cc94c666-25e6-4d20-81a8-a6e77d84960c", "drop:1c59a315-7720-42c8-87ef-32cef6c26af0", "drop:59313c39-55a8-4ea6-a019-149220ec4fc2", "drop:d94522ce-e474-4f02-8f6a-73d01cdba185", "drop:04383fde-eec8-4255-ae75-6f132e20f9ed", "drop:6e2e25eb-7f74-4944-8e29-c666e1adac6b", "drop:c95c3fb6-fb82-4fb9-9e1d-9209aace7f34", "drop:a81ac703-0b94-4b30-8dea-f01fbe39720f", "drop:0db6d7f7-de1a-4c22-9835-8b51ab2b1781", "drop:6639ffe0-acdd-40cc-a6f5-081c6e09df04", "drop:b5812c99-494b-4730-a7b2-cca7bff83c91", "drop:6ee455be-25b0-4373-9246-49c25dd405a5", "drop:52401fa3-013b-4bce-86f2-7ccf0c0d318b", "drop:309e4a60-9645-45b3-920d-2d0049c7a50a", "drop:90a99b38-2ac9-4814-b706-d6f1ead39200", "drop:3aa72f74-86fe-4b32-83d2-e8845fca88e3", "drop:c0c88600-4eeb-47f8-b2b5-7d88d2f203fc", "drop:64d78350-ac88-4272-b87e-aee3025667cf", "drop:eff796fe-d60c-4c91-8eec-5c6fb86071a1", "drop:361959e1-3ed0-4440-91e9-9d29b9e69fa2", "drop:0e5839b6-7200-437f-b335-cb93e38ad2fc", "drop:b44649b5-5200-4f0a-99bf-10679fb02e2a", "drop:9c5ca555-cf71-4995-b1a3-635c0403a2cc", "drop:d070b0bb-cf79-4d15-af15-ea62c9fe6812", "drop:4a7c2399-410a-47c5-925b-e839f742b3f3", "drop:7b881f5f-b818-4709-85b6-9acdb33b746f", "drop:48badfee-532b-43d3-999e-106e90ac463c", "drop:eaad52d1-fb56-4796-9099-a8f0567b4161", "drop:d81d0bd9-0f75-4e88-93d1-847ee4000a36", "drop:53825b9d-5f79-4437-902c-fa49d183f96e", "drop:eefc7c3a-ca5c-401b-94d2-a84fcec35946", "drop:6ae0fe84-bd7c-4486-acf4-b56d67f9858d", "drop:6adb1a90-56ab-48d4-8a5a-16858ef34690" ] }, "klokan_cs": { "source_urls": [ "https://huggingface.co/datasets/hynky/klokan-qa" ], "count": 150, "task_ids": [ "klokan_cs:8", "klokan_cs:14", "klokan_cs:22", "klokan_cs:25", "klokan_cs:33", "klokan_cs:34", "klokan_cs:38", "klokan_cs:40", "klokan_cs:46", "klokan_cs:50", "klokan_cs:64", "klokan_cs:75", "klokan_cs:77", "klokan_cs:92", "klokan_cs:99", "klokan_cs:101", "klokan_cs:109", "klokan_cs:110", "klokan_cs:117", "klokan_cs:118", "klokan_cs:119", "klokan_cs:126", "klokan_cs:134", "klokan_cs:135", "klokan_cs:138", "klokan_cs:140", "klokan_cs:143", "klokan_cs:144", "klokan_cs:158", "klokan_cs:161", "klokan_cs:168", "klokan_cs:179", "klokan_cs:194", "klokan_cs:203", "klokan_cs:205", "klokan_cs:222", "klokan_cs:224", "klokan_cs:226", "klokan_cs:231", "klokan_cs:235", "klokan_cs:239", "klokan_cs:240", "klokan_cs:242", "klokan_cs:256", "klokan_cs:263", "klokan_cs:268", "klokan_cs:270", "klokan_cs:279", "klokan_cs:281", "klokan_cs:286", "klokan_cs:293", "klokan_cs:294", "klokan_cs:302", "klokan_cs:306", "klokan_cs:309", "klokan_cs:311", "klokan_cs:316", "klokan_cs:334", "klokan_cs:335", "klokan_cs:346", "klokan_cs:353", "klokan_cs:360", "klokan_cs:367", "klokan_cs:373", "klokan_cs:374", "klokan_cs:377", "klokan_cs:379", "klokan_cs:380", "klokan_cs:390", "klokan_cs:393", "klokan_cs:397", "klokan_cs:404", "klokan_cs:405", "klokan_cs:406", "klokan_cs:417", "klokan_cs:418", "klokan_cs:424", "klokan_cs:425", "klokan_cs:429", "klokan_cs:434", "klokan_cs:451", "klokan_cs:452", "klokan_cs:464", "klokan_cs:465", "klokan_cs:467", "klokan_cs:475", "klokan_cs:479", "klokan_cs:482", "klokan_cs:493", "klokan_cs:494", "klokan_cs:501", "klokan_cs:504", "klokan_cs:508", "klokan_cs:513", "klokan_cs:543", "klokan_cs:549", "klokan_cs:550", "klokan_cs:552", "klokan_cs:553", "klokan_cs:559", "klokan_cs:570", "klokan_cs:574", "klokan_cs:579", "klokan_cs:585", "klokan_cs:588", "klokan_cs:593", "klokan_cs:598", "klokan_cs:601", "klokan_cs:620", "klokan_cs:621", "klokan_cs:624", "klokan_cs:630", "klokan_cs:634", "klokan_cs:635", "klokan_cs:642", "klokan_cs:643", "klokan_cs:650", "klokan_cs:651", "klokan_cs:653", "klokan_cs:654", "klokan_cs:661", "klokan_cs:663", "klokan_cs:664", "klokan_cs:670", "klokan_cs:674", "klokan_cs:675", "klokan_cs:688", "klokan_cs:690", "klokan_cs:691", "klokan_cs:692", "klokan_cs:693", "klokan_cs:694", "klokan_cs:697", "klokan_cs:702", "klokan_cs:706", "klokan_cs:707", "klokan_cs:716", "klokan_cs:734", "klokan_cs:738", "klokan_cs:741", "klokan_cs:742", "klokan_cs:752", "klokan_cs:762", "klokan_cs:763", "klokan_cs:780", "klokan_cs:782", "klokan_cs:786", "klokan_cs:792", "klokan_cs:794", "klokan_cs:799" ] } }, "frozen_input_files": [ { "name": "tasks.jsonl", "sha256": "d621f04e665d368608f16c50c98fbabd054111489c56dcd2c9e33759511a2664", "task_count": 1419, "counts_by_type": { "gpqa": 198, "math500": 500, "aime": 60, "ifeval": 541, "hotpot": 120 }, "set": "main" }, { "name": "tasks-cz.jsonl", "sha256": "0c42fac61ec606d92d4aa0e679dd0b89c152fb4b2d2d754f422b3555234de095", "task_count": 1065, "counts_by_type": { "cz_cermat": 649, "cz_math": 126, "cz_mmlu": 290 }, "set": "main" }, { "name": "tasks-code.jsonl", "sha256": "3323a0bcca6e0e2c47954f92e62aa8ce699e58af482d64b9d27791845f148c61", "task_count": 60, "counts_by_type": { "lcb": 60 }, "set": "main" }, { "name": "tasks-a8.jsonl", "sha256": "5629d759c7c3278af0f5df77e72d5448dc0db1a8b0c56e71e7e7fa00c8472121", "task_count": 1455, "counts_by_type": { "bbh": 405, "mmlupro": 350, "belebele_cs": 200, "cz_agree": 150, "drop": 200, "klokan_cs": 150 }, "set": "A8" } ], "generation_presets": { "qwen-think": { "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "_think": true }, "qwen-nothink": { "temperature": 0.7, "top_p": 0.8, "top_k": 20, "min_p": 0.0, "presence_penalty": 1.5, "_think": false }, "gemma-think": { "temperature": 1.0, "top_p": 0.95, "top_k": 64, "skip_special_tokens": false, "_think": true }, "gemma-nothink": { "temperature": 1.0, "top_p": 0.95, "top_k": 64, "skip_special_tokens": false, "_think": false }, "gptoss": { "temperature": 1.0, "top_p": 1.0, "_think": null }, "gptoss-high": { "temperature": 1.0, "top_p": 1.0, "reasoning_effort": "high", "_think": null }, "mimo-think": { "temperature": 1.0, "top_p": 0.95, "_think": true } }, "sampling_notes": { "main": "Typically seeds 1-4 or 1-8; scaling experiments up to 32.", "selection_seed": 42, "A8_selection_order": [ "bbh", "mmlupro", "belebele_cs", "cz_agree", "drop", "klokan_cs" ], "default_context_tokens": 40960, "default_output_cap_tokens": 38000, "reruns": "Some truncated outputs were regenerated with longer limits. Defaults alone do not reproduce those runs.", "router_evaluation": "Selection on repetition group A, evaluation on B; cross-validation also separates unique questions. A8 evaluated after router lock.", "code_evaluation": "Public tests for selecting candidates; separate hidden tests for final scoring." }, "observed_configuration_examples": { "gemma4-26b/nothink": { "config": "gemma4-26b", "branch": "nothink", "model": "google/gemma-4-26B-A4B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-nothink", "max_tokens": 38000, "seed": 1 }, "gemma4-26b/nothink128k": { "config": "gemma4-26b", "branch": "nothink128k", "model": "google/gemma-4-26B-A4B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-nothink", "max_tokens": null, "seed": 2 }, "gemma4-26b/think": { "config": "gemma4-26b", "branch": "think", "model": "google/gemma-4-26B-A4B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-think", "max_tokens": 38000, "seed": 1 }, "gemma4-26b/think128k": { "config": "gemma4-26b", "branch": "think128k", "model": "google/gemma-4-26B-A4B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-think", "max_tokens": null, "seed": 2 }, "gemma4-26b-long/nothink": { "config": "gemma4-26b-long", "branch": "nothink", "model": "google/gemma-4-26B-A4B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-nothink", "max_tokens": 38000, "seed": 1 }, "gemma4-31b/think": { "config": "gemma4-31b", "branch": "think", "model": "google/gemma-4-31B-it", "engine": "vllm", "engine_version": "0.22.1", "preset": "gemma-think", "max_tokens": 38000, "seed": 1 }, "gptoss-120b/default": { "config": "gptoss-120b", "branch": "default", "model": "/appl/local/laifs/models/gpt-oss-120b", "engine": "vllm", "engine_version": "0.22.1", "preset": "gptoss", "max_tokens": 38000, "seed": 1 }, "gptoss-120b-hi/high": { "config": "gptoss-120b-hi", "branch": "high", "model": "/appl/local/laifs/models/gpt-oss-120b", "engine": "vllm", "engine_version": "0.22.1", "preset": "gptoss-high", "max_tokens": 38000, "seed": 1 }, "gptoss-120b-hi/high128k": { "config": "gptoss-120b-hi", "branch": "high128k", "model": "/appl/local/laifs/models/gpt-oss-120b", "engine": "vllm", "engine_version": "0.22.1", "preset": "gptoss-high", "max_tokens": null, "seed": 1 }, "gptoss-120b-long/default": { "config": "gptoss-120b-long", "branch": "default", "model": "/appl/local/laifs/models/gpt-oss-120b", "engine": "vllm", "engine_version": "0.22.1", "preset": "gptoss", "max_tokens": 38000, "seed": 1 }, "gptoss-120b-long/default128k": { "config": "gptoss-120b-long", "branch": "default128k", "model": "/appl/local/laifs/models/gpt-oss-120b", "engine": "vllm", "engine_version": "0.22.1", "preset": "gptoss", "max_tokens": null, "seed": 1 }, "mimo-flash-gguf/think": { "config": "mimo-flash-gguf", "branch": "think", "model": "mimo-flash-gguf", "engine": "llama.cpp", "engine_version": "c2a9e16", "preset": "mimo-think", "max_tokens": 38000, "seed": 1 }, "mimo-flash-gguf/think128k": { "config": "mimo-flash-gguf", "branch": "think128k", "model": "mimo-flash-gguf", "engine": "llama.cpp", "engine_version": "c2a9e16", "preset": "mimo-think", "max_tokens": null, "seed": 1 }, "mimo-pro-bpw35/think": { "config": "mimo-pro-bpw35", "branch": "think", "model": "mimo-pro-bpw35", "engine": "llama.cpp", "engine_version": "c2a9e16", "preset": "mimo-think", "max_tokens": 38000, "seed": 1 }, "mimo-pro-bpw35/think128k": { "config": "mimo-pro-bpw35", "branch": "think128k", "model": "mimo-pro-bpw35", "engine": "llama.cpp", "engine_version": "c2a9e16", "preset": "mimo-think", "max_tokens": null, "seed": 1 }, "mimo-pro-mxfp4/think": { "config": "mimo-pro-mxfp4", "branch": "think", "model": "mimo-pro-mxfp4", "engine": "llama.cpp", "engine_version": "c2a9e16", "preset": "mimo-think", "max_tokens": 38000, "seed": 1 }, "qwen36-27b/think": { "config": "qwen36-27b", "branch": "think", "model": "Qwen/Qwen3.6-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": 38000, "seed": 1 }, "qwen36-27b/think128k": { "config": "qwen36-27b", "branch": "think128k", "model": "Qwen/Qwen3.6-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": null, "seed": 1 }, "qwen36-35b/nothink": { "config": "qwen36-35b", "branch": "nothink", "model": "Qwen/Qwen3.6-35B-A3B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-nothink", "max_tokens": 38000, "seed": 1 }, "qwen36-35b/nothink128k": { "config": "qwen36-35b", "branch": "nothink128k", "model": "Qwen/Qwen3.6-35B-A3B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-nothink", "max_tokens": null, "seed": 3 }, "qwen36-35b/think": { "config": "qwen36-35b", "branch": "think", "model": "Qwen/Qwen3.6-35B-A3B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": 38000, "seed": 1 }, "qwen36-35b/think128k": { "config": "qwen36-35b", "branch": "think128k", "model": "Qwen/Qwen3.6-35B-A3B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": null, "seed": 1 }, "qwen36-35b-long/nothink": { "config": "qwen36-35b-long", "branch": "nothink", "model": "Qwen/Qwen3.6-35B-A3B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-nothink", "max_tokens": 38000, "seed": 1 }, "qwen38-27b/nothink": { "config": "qwen38-27b", "branch": "nothink", "model": "Qwen/Qwen3.8-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-nothink", "max_tokens": 38000, "seed": 1 }, "qwen38-27b/think": { "config": "qwen38-27b", "branch": "think", "model": "Qwen/Qwen3.8-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": 38000, "seed": 1 }, "qwen38-27b/think128k": { "config": "qwen38-27b", "branch": "think128k", "model": "Qwen/Qwen3.8-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": null, "seed": 1 }, "qwen38-27b-long/nothink": { "config": "qwen38-27b-long", "branch": "nothink", "model": "Qwen/Qwen3.8-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-nothink", "max_tokens": 38000, "seed": 1 }, "qwen38-27b-syn/think": { "config": "qwen38-27b-syn", "branch": "think", "model": "Qwen/Qwen3.8-27B", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": 37860, "seed": 1 }, "thinkingcap-27b/think": { "config": "thinkingcap-27b", "branch": "think", "model": "/scratch/project_465003187/models/thinkingcap", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": 38000, "seed": 1 }, "thinkingcap-27b/think128k": { "config": "thinkingcap-27b", "branch": "think128k", "model": "/scratch/project_465003187/models/thinkingcap", "engine": "vllm", "engine_version": "0.22.1", "preset": "qwen-think", "max_tokens": null, "seed": 1 } } }