{ "design": "5 uniformly sampled prompts per model, seed 2026091107; model names and Qwen labels hidden during assistant review. Same assistant ran the benchmark, so this is not an independent human validation. Primary scores unchanged.", "planned": 100, "paired_available": 100, "agreement": 92, "disagreement": 6, "uncertain": 2, "details": [ { "audit_id": "A001", "model": "HuggingFaceTB/SmolLM-135M", "pair_id": "P039", "text": "The photograph on the wall, taken in 1991, shows a girl in a dress holding a bottle and a book.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid syntax and subject-verb agreement.", "agrees": true }, { "audit_id": "A002", "model": "state-spaces/mamba-130m-hf", "pair_id": "P041", "text": "The wind from the sea is coming in.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Simple, grammatically correct sentence with clear subject and verb.", "agrees": true }, { "audit_id": "A003", "model": "distilbert/distilgpt2", "pair_id": "P067", "text": "The children who were waiting outside the house for their father's family home on the grounds of a home in the middle of the morning were told to leave the building.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Syntactically complete sentence, despite semantic confusion in the prepositional phrase.", "agrees": true }, { "audit_id": "A004", "model": "EleutherAI/gpt-neo-125m", "pair_id": "P007", "text": "The tired driver who is making a dashcam video of the accident, a man who was hit by a truck and killed by a drunk driver, has said he will not be able to drive again.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with a complex subject and finite verb 'has said'.", "agrees": true }, { "audit_id": "A005", "model": "SlayerLab/new-training-run-ph-d7e9fe9b", "pair_id": "P033", "text": "The book on the shelf dochylente regionkse.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Nonsensical word 'dochylente regionkse'; not valid English syntax.", "agrees": true }, { "audit_id": "A006", "model": "SlayerLab/GoLLeM-110M-PL-v2", "pair_id": "P041", "text": "The wind from the seasons are very because for the long.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Subject-verb disagreement and incoherent prepositional phrase.", "agrees": true }, { "audit_id": "A007", "model": "speakleash/Bielik-11B-v3.0-Instruct", "pair_id": "P005", "text": "The children in the garden were not happy about this.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject and finite verb.", "agrees": true }, { "audit_id": "A008", "model": "SlayerLab/new-training-run-ph-d7e9fe9b", "pair_id": "P060", "text": "Whenever I hear that song,\n tychce gon całkowite comm oraz białe w cywilizacje odezwolenie przewidzę.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Contains non-English text and lacks a main clause for the subordinate prefix.", "agrees": true }, { "audit_id": "A009", "model": "SlayerLab/GoLLeM-45M-PL", "pair_id": "P037", "text": "The last piece of breadly, the most of the greatest to the 8/12.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Invalid word 'breadly' and incoherent prepositional phrase structure.", "agrees": true }, { "audit_id": "A010", "model": "SlayerLab/new-training-run-ph-d7e9fe9b", "pair_id": "P055", "text": "While the children were sleeping, z razem wszystko komuniki nastronnego klucę, z kluczem widziany połączenia mandaty, które jak pierwości pojazdło przebyczają i dlatego swego pora.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Polish text is not valid English; syntactically incoherent in English.", "agrees": true }, { "audit_id": "A011", "model": "EleutherAI/pythia-160m", "pair_id": "P002", "text": "My older brother.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Bare noun phrase; not a complete sentence.", "agrees": true }, { "audit_id": "A012", "model": "EleutherAI/gpt-neo-125m", "pair_id": "P063", "text": "The people we met yesterday were amazing.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject and predicate.", "agrees": true }, { "audit_id": "A013", "model": "McGill-NLP/TLM-100M", "pair_id": "P094", "text": "My mother and her sister, a very young girl from the northern states, had a beautiful, wonderful-looking face that was very expressive of the natural beauty of the earth.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with correct syntax and verb usage.", "agrees": true }, { "audit_id": "A014", "model": "MBZUAI/LaMini-GPT-124M", "pair_id": "P092", "text": "Neither the rain nor the wind.)", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment with a stray closing parenthesis and no main verb.", "agrees": true }, { "audit_id": "A015", "model": "openai-community/gpt2", "pair_id": "P047", "text": "The trees along the road were mostly empty.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Simple, grammatically correct sentence with a clear subject and predicate.", "agrees": true }, { "audit_id": "A016", "model": "SlayerLab/bdh-25m-pl", "pair_id": "P095", "text": "Not only the price but also the quality i lampis od Salon zdobycia się także bycia oświadczenia.", "assistant_pass": false, "assistant_reason": "Coordinated English noun phrase followed by malformed Polish, with no complete English predicate.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Noun phrase followed by Polish fragment; no main clause.", "agrees": true }, { "audit_id": "A017", "model": "SlayerLab/bdh-25m-pl", "pair_id": "P083", "text": "We did not know whether to mego bramka deportu są specyfikę w Polsce.", "assistant_pass": false, "assistant_reason": "The English prefix continues into malformed mixed-language text, not a complete English sentence.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Mixed languages and incoherent syntax; no valid main clause.", "agrees": true }, { "audit_id": "A018", "model": "openai-community/gpt2", "pair_id": "P025", "text": "During the long winter of the 19th century, American colonists settled in the vicinity of Fort Bragg, New Jersey.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid syntax and agreement.", "agrees": true }, { "audit_id": "A019", "model": "SlayerLab/slayer-scratch", "pair_id": "P087", "text": "Nobody understood whyd.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "'whyd' is a morphological error; the sentence is incomplete.", "agrees": true }, { "audit_id": "A020", "model": "distilbert/distilgpt2", "pair_id": "P092", "text": "Neither the rain nor the wind will continue to rain.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid subject-verb agreement; semantically odd but syntactically sound.", "agrees": true }, { "audit_id": "A021", "model": "distilbert/distilgpt2", "pair_id": "P020", "text": "Outside the city centre was closed for more than a day on Saturday, April 25, 2009 by a special election for the city council and the government.", "assistant_pass": null, "assistant_reason": "Locative phrase as subject is marginal; context could license a place reading.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid passive construction.", "agrees": null }, { "audit_id": "A022", "model": "SlayerLab/GoLLeM-110M-PL-v3", "pair_id": "P050", "text": "The sound of falling rains, and it for the slow bills someone (Price in the causes) to think all in the situation, and the recrubble to the end of our health.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent syntax, missing main verb, nonsensical structure.", "agrees": true }, { "audit_id": "A023", "model": "SlayerLab/GoLLeM-45M-PL", "pair_id": "P087", "text": "Nobody understood why in 25s.", "assistant_pass": true, "assistant_reason": "Understood why can take a time adjunct; semantic oddity is not a syntax error.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment; 'why in 25s' lacks a main clause verb or object.", "agrees": false }, { "audit_id": "A024", "model": "SlayerLab/GoLLeM-110M-PL-v2", "pair_id": "P071", "text": "She carefully opened with other time-content and company of the children’s results in the colors of results.", "assistant_pass": true, "assistant_reason": "Opened with can be intransitive; semantic oddity is not a syntax error.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Nonsensical syntax and word order; syntactically complete but grammatically invalid.", "agrees": false }, { "audit_id": "A025", "model": "HuggingFaceTB/SmolLM-135M", "pair_id": "P002", "text": "My older brother, who works as a security guard, was having trouble catching his bus today.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject, relative clause, and main verb.", "agrees": true }, { "audit_id": "A026", "model": "McGill-NLP/TLM-100M", "pair_id": "P070", "text": "The question that nobody could answer is: how much time do you spend on a daily basis?”", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid syntax.", "agrees": true }, { "audit_id": "A027", "model": "MBZUAI/LaMini-GPT-124M", "pair_id": "P046", "text": "The dog behind the fence, but it needs to be kept in a safe place.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment followed by a clause; no main verb for the subject.", "agrees": true }, { "audit_id": "A028", "model": "facebook/opt-125m", "pair_id": "P059", "text": "Unless something changes, I'd be in for it.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject 'I' and finite verb 'be' (in conditional); syntax is valid.", "agrees": true }, { "audit_id": "A029", "model": "openai-community/gpt2", "pair_id": "P054", "text": "If you have enough time, I suggest you start off by watching a good college movie.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct conditional sentence with clear main clause.", "agrees": true }, { "audit_id": "A030", "model": "state-spaces/mamba-130m-hf", "pair_id": "P006", "text": "Our closest neighbours are the other species, as well as a few small trees and an occasional bird.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct sentence with a compound predicate and clear subject-verb agreement.", "agrees": true }, { "audit_id": "A031", "model": "SlayerLab/bdh-25m-pl", "pair_id": "P067", "text": "The children who were waiting outsiden-\nry, co położyło się w przypadku gospodarki na stan mnie inkwestii.", "assistant_pass": false, "assistant_reason": "The English prefix continues into malformed mixed-language text, not a complete English sentence.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Subject is followed by a fragment, lacking a main finite verb.", "agrees": true }, { "audit_id": "A032", "model": "SlayerLab/goLLeM-110M-PL-SFT-merged", "pair_id": "P010", "text": "Everyone in the room, resume iPhone'y i iPhone'y i iPhone'y — tylko nowe dane można pobrać z biblioteki.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Mixed languages and broken syntax; 'resume iPhone'y' is ungrammatical.", "agrees": true }, { "audit_id": "A033", "model": "state-spaces/mamba-130m-hf", "pair_id": "P085", "text": "It is important to make the distinction between the content and the performance of a service.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with infinitive phrase as subject complement.", "agrees": true }, { "audit_id": "A034", "model": "SlayerLab/bdh-25m-pl", "pair_id": "P034", "text": "These small detailstanersza i podkreślaniu szlakowego i otrzymywanego amerykańskiego praca za wszystkie kontrole i dopuszczane przedstawiane rządowe przez długotrwałe przedsiębiorstwo sprzętu przez włoski zastanowiła się na ponad 1400 r., ale osób, w kierunku powstałego stanu i przewody wymaga większej kraju.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Nonsensical string of words lacking syntactic structure and complete clause.", "agrees": true }, { "audit_id": "A035", "model": "SlayerLab/pollock-mini-lm-125m", "pair_id": "P096", "text": "The first and the second chapter of the \"Later Chronicles\" of Homer's Odyssey.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Noun phrase only; lacks a finite verb to form a complete sentence.", "agrees": true }, { "audit_id": "A036", "model": "distilbert/distilgpt2", "pair_id": "P002", "text": "My older brother and he said hello to me.", "assistant_pass": true, "assistant_reason": "My older brother and he is a coordinated subject; past-tense said agrees.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "The subject 'My older brother and he' is syntactically invalid; 'he' is a pronoun, not a noun.", "agrees": false }, { "audit_id": "A037", "model": "SlayerLab/pollock-mini-lm-125m", "pair_id": "P043", "text": "The water in the riverbed is deep.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject, verb, and predicate adjective.", "agrees": true }, { "audit_id": "A038", "model": "facebook/opt-125m", "pair_id": "P003", "text": "The new teacher is expected to be in the state for six weeks.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct sentence with clear main clause.", "agrees": true }, { "audit_id": "A039", "model": "SlayerLab/pollock-mini-lm-125m", "pair_id": "P016", "text": "Under the wooden bridge of the same name, a number of \nthose in the first and second class of cases will show, in \na number of the first class, that the names of the men \nand women of the first class are written in this style, \nand are all of the same order.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically valid, albeit complex and repetitive, sentence structure.", "agrees": true }, { "audit_id": "A040", "model": "distilbert/distilgpt2", "pair_id": "P028", "text": "After several months after announcing that Uber’s future in the U.S. has been delayed, the startup says it is planning to open a new office in Santa Clara, California.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence.", "agrees": true }, { "audit_id": "A041", "model": "HuggingFaceTB/SmolLM2-135M", "pair_id": "P079", "text": "The little boy wanted his mother to come and visit him.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject 'boy' and verb 'wanted'.", "agrees": true }, { "audit_id": "A042", "model": "speakleash/Bielik-11B-v3.0-Instruct", "pair_id": "P090", "text": "It turned out that beetles that had been exposed to the drug as larvae were less likely to develop the disease as adults.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "The sentence is grammatically correct and syntactically complete.", "agrees": true }, { "audit_id": "A043", "model": "speakleash/Bielik-11B-v3.0-Instruct", "pair_id": "P062", "text": "The book that you gave me on your last visit, the one about the history of the United States, was really interesting.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence.", "agrees": true }, { "audit_id": "A044", "model": "SlayerLab/GoLLeM-110M-PL-v2", "pair_id": "P025", "text": "During the long winter report in the complete results for many from the game to the consecularly induced to from the results for many from the bunch and their more in the game innominate and the best of the fashion of the effects in Poland.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent syntax and missing main verb.", "agrees": true }, { "audit_id": "A045", "model": "SlayerLab/GoLLeM-110M-PL-v3", "pair_id": "P088", "text": "I wonder how does it golf that after time my will watch they wonder in there going to this twitter.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incorrect word order in embedded question 'how does it golf'.", "agrees": true }, { "audit_id": "A046", "model": "EleutherAI/pythia-70m", "pair_id": "P066", "text": "The reason why we came here, though, is that a place where we can't be here for the world.", "assistant_pass": false, "assistant_reason": "That introduces a complement without a finite main predicate.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incorrect predicate structure; 'is that a place' is syntactically flawed.", "agrees": true }, { "audit_id": "A047", "model": "SlayerLab/new-training-run-ph-d7e9fe9b", "pair_id": "P052", "text": "Although the road was long, netom-vert (jednak) to nie stara polei -to poetyki off the -taper do the e wszystko wass długogowych the the t le będzie zł tej bez możliwych mają pewnym całej i część.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent mixed-language text; no valid main clause.", "agrees": true }, { "audit_id": "A048", "model": "HuggingFaceTB/SmolLM-135M", "pair_id": "P054", "text": "If you have enough time, you may want to try two different methods for your test.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence.", "agrees": true }, { "audit_id": "A049", "model": "EleutherAI/gpt-neo-125m", "pair_id": "P021", "text": "Early in the morning he got back on the bike.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Syntactically correct and complete sentence.", "agrees": true }, { "audit_id": "A050", "model": "SlayerLab/goLLeM-110M-PL-SFT-merged", "pair_id": "P007", "text": "The tired driver — domyślne kroki.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Noun phrase followed by unrelated Polish text; no main clause.", "agrees": true }, { "audit_id": "A051", "model": "SlayerLab/goLLeM-110M-PL-SFT-merged", "pair_id": "P092", "text": "Neither the rain nor the wind whicheshile z wagami i model uczy się z cech.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Mixed languages and incoherent syntax; not a valid English sentence.", "agrees": true }, { "audit_id": "A052", "model": "HuggingFaceTB/SmolLM2-135M", "pair_id": "P022", "text": "Last summer, the world’s major cities had a population of 280 million, 20 percent of the world’s population, and 90 percent of the world’s population was in urban areas.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence with clear subject and verb.", "agrees": true }, { "audit_id": "A053", "model": "MBZUAI/LaMini-GPT-124M", "pair_id": "P100", "text": "Most of the people in the village where the fire occurred.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Noun phrase with relative clause, lacks a main verb.", "agrees": true }, { "audit_id": "A054", "model": "HuggingFaceTB/SmolLM2-135M", "pair_id": "P060", "text": "Whenever I hear that song, I begin to think that it is a song I like, and I always think of my favorite songs.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete and grammatically correct sentence.", "agrees": true }, { "audit_id": "A055", "model": "HuggingFaceTB/SmolLM-135M", "pair_id": "P005", "text": "The children in the garden were delighted and it was a good time to go on a hunt for bugs, frogs and a few other things to eat.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with a finite verb and valid compound predicate structure.", "agrees": true }, { "audit_id": "A056", "model": "facebook/opt-125m", "pair_id": "P083", "text": "We did not know whether to send him home or not.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence.", "agrees": true }, { "audit_id": "A057", "model": "EleutherAI/pythia-160m", "pair_id": "P010", "text": "Everyone in the room, and I just wanted to do some things with a little time.\"", "assistant_pass": true, "assistant_reason": "Everyone in the room and I is a coordinated subject; comma alone is harmless.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment with dangling conjunction and unclosed quote.", "agrees": false }, { "audit_id": "A058", "model": "McGill-NLP/TLM-100M", "pair_id": "P001", "text": "The young woman was in a coma for a while.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid sentence structure with subject, verb, and prepositional phrase.", "agrees": true }, { "audit_id": "A059", "model": "EleutherAI/pythia-160m", "pair_id": "P090", "text": "It turned out that our first trip to the US during the summer of 2016 was a farce, so we wanted to have a tour of our new home.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid subordinate clause.", "agrees": true }, { "audit_id": "A060", "model": "EleutherAI/pythia-160m", "pair_id": "P059", "text": "Unless something changes, you can contact me and we can always talk again.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid conditional sentence with a complete main clause and subordinate clause.", "agrees": true }, { "audit_id": "A061", "model": "SlayerLab/new-training-run-ph-d7e9fe9b", "pair_id": "P048", "text": "The sun above the mountains o sposób porównania stytu ostatniego umrzełych ostatniej\n możesz styl.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Mixed languages; Polish fragment is not a complete clause.", "agrees": true }, { "audit_id": "A062", "model": "McGill-NLP/TLM-100M", "pair_id": "P009", "text": "Two old friends, with their families and friends, gathered around a beautiful tree in the parking lot of the local library which is a huge place.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with a clear subject and finite verb.", "agrees": true }, { "audit_id": "A063", "model": "EleutherAI/pythia-70m", "pair_id": "P033", "text": "The book on the shelf will include a brief list of articles that will be published in the book.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct sentence with proper structure.", "agrees": true }, { "audit_id": "A064", "model": "SlayerLab/slayer-scratch", "pair_id": "P055", "text": "While the children were sleeping, a nie ma żadnego sensu w stanie po prostu być.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Mixed languages; Polish fragment lacks a main clause.", "agrees": true }, { "audit_id": "A065", "model": "SlayerLab/GoLLeM-110M-PL-v3", "pair_id": "P028", "text": "After several months from the either with avecary of the didn' diverse and contacts to be more in the continue to its in the considers of the current and continue to the everyone of the process of their research to the education of the out of the results.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent word salad with no finite verb or syntactic structure.", "agrees": true }, { "audit_id": "A066", "model": "SlayerLab/slayer-scratch", "pair_id": "P011", "text": "In the middle of the forest, the the the the the and the the the the the the the the snow the s the the the.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Repetitive, non-lexical tokens result in a syntactically invalid fragment.", "agrees": true }, { "audit_id": "A067", "model": "EleutherAI/pythia-160m", "pair_id": "P064", "text": "The house where I grew up, and what I do have is a lot of good books and a very good teacher.\"", "assistant_pass": false, "assistant_reason": "Conjoined subject with singular is; also fragmentary coordination.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragmented sentence with mismatched conjunction and incomplete thought.", "agrees": true }, { "audit_id": "A068", "model": "state-spaces/mamba-130m-hf", "pair_id": "P014", "text": "On the kitchen table he was still holding the magazine he had just finished reading, the one she had given him over the previous day.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid sentence with a prepositional phrase and main clause.", "agrees": true }, { "audit_id": "A069", "model": "SlayerLab/goLLeM-110M-PL-SFT-merged", "pair_id": "P054", "text": "If you have enough time, well have evaluation, invest IDMS.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragmented structure; 'well have evaluation' is not a valid clause.", "agrees": true }, { "audit_id": "A070", "model": "SlayerLab/GoLLeM-45M-PL", "pair_id": "P074", "text": "They had already finished.", "assistant_pass": false, "assistant_reason": "Only the supplied prefix plus a period; no completion under this rubric.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with finite verb 'had finished'.", "agrees": false }, { "audit_id": "A071", "model": "SlayerLab/GoLLeM-110M-PL-v3", "pair_id": "P065", "text": "The woman whose car broke down to the india.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "'Broke down to the India' is syntactically incorrect; 'India' should be 'India' without 'the' or rephrased.", "agrees": true }, { "audit_id": "A072", "model": "speakleash/Bielik-11B-v3.0-Instruct", "pair_id": "P048", "text": "The sun above the mountains is a symbol of harmonious union and creative energy.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid sentence with subject and predicate.", "agrees": true }, { "audit_id": "A073", "model": "EleutherAI/gpt-neo-125m", "pair_id": "P030", "text": "By the beginning of September, the federal government and the American people began to mobilize around the world to organize a protest demonstration in New Hampshire, Massachusetts, Rhode Island, Connecticut and Connecticut and New England.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid sentence with subject and compound predicate.", "agrees": true }, { "audit_id": "A074", "model": "SlayerLab/GoLLeM-45M-PL", "pair_id": "P038", "text": "All the windows in the house of the first shared, the first shared bearing the most important cummit.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragmented syntax; 'of the first shared' is not a valid clause structure.", "agrees": true }, { "audit_id": "A075", "model": "SlayerLab/pollock-mini-lm-125m", "pair_id": "P010", "text": "Everyone in the room is saying thank you but there is a loud crash.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct compound sentence with proper agreement.", "agrees": true }, { "audit_id": "A076", "model": "SlayerLab/bdh-25m-pl", "pair_id": "P029", "text": "Every Sunday afternooniczne.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "The suffix 'iczne' creates a non-word, resulting in a syntactic error.", "agrees": true }, { "audit_id": "A077", "model": "MBZUAI/LaMini-GPT-124M", "pair_id": "P040", "text": "Several empty bottles), which is also the reason for the lack of the standard equipment to support the needs of all patrons.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment with unmatched parenthesis and no main verb.", "agrees": true }, { "audit_id": "A078", "model": "facebook/opt-125m", "pair_id": "P098", "text": "Some of the guests at the party include former President George W. Bush and former United States Secretary of State Condoleezza Rice.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject, verb, and object.", "agrees": true }, { "audit_id": "A079", "model": "MBZUAI/LaMini-GPT-124M", "pair_id": "P059", "text": "Unless something changes, but it is possible to achieve the desired outcome.", "assistant_pass": false, "assistant_reason": "But incorrectly joins an unless clause to its main clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incorrect conjunction 'but' following a conditional clause; syntactically invalid.", "agrees": true }, { "audit_id": "A080", "model": "speakleash/Bielik-11B-v3.0-Instruct", "pair_id": "P030", "text": "By the beginning of September, the team was preparing to return to the United States.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject and finite verb.", "agrees": true }, { "audit_id": "A081", "model": "openai-community/gpt2", "pair_id": "P026", "text": "On the following day, the United States government informed the Senate that a special prosecutor had closed the investigation into the alleged role of the CIA in the death of Osama bin Laden.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct complex sentence with clear subject and predicate.", "agrees": true }, { "audit_id": "A082", "model": "SlayerLab/goLLeM-110M-PL-SFT-merged", "pair_id": "P073", "text": "We were looking for aval, costs i cool, cool.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Contains non-words and lacks a clear object; incomplete and ungrammatical.", "agrees": true }, { "audit_id": "A083", "model": "SlayerLab/GoLLeM-110M-PL-v2", "pair_id": "P028", "text": "After several months from Unverked with a revision of resistance of resistant and learning to their \"sugestive design\" in the considers of \"after increase country\" (ang. \"secret\") – język australijski, używany w wielu krajach na terenie stanu Alaska.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Nonsensical syntax and mixed languages; 'from Unverked' is ungrammatical.", "agrees": true }, { "audit_id": "A084", "model": "SlayerLab/GoLLeM-110M-PL-v2", "pair_id": "P072", "text": "He decided to the even attendy, afternoon to the think think, we all the same invite, some time!", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent syntax; no valid clause.", "agrees": true }, { "audit_id": "A085", "model": "HuggingFaceTB/SmolLM-135M", "pair_id": "P086", "text": "The problem is that you can’t get these new data centers online until the Internet is fully up and running after about 10 years.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Grammatically correct and complete sentence.", "agrees": true }, { "audit_id": "A086", "model": "HuggingFaceTB/SmolLM2-135M", "pair_id": "P035", "text": "The key to the front door was the small, round, white, and white-painted, metal door.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject and predicate.", "agrees": true }, { "audit_id": "A087", "model": "HuggingFaceTB/SmolLM2-135M", "pair_id": "P054", "text": "If you have enough time, let’s go into detail what we want to know about this.", "assistant_pass": false, "assistant_reason": "Go into detail needs a linking preposition before the what clause.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with conditional clause and imperative main clause.", "agrees": false }, { "audit_id": "A088", "model": "EleutherAI/pythia-70m", "pair_id": "P084", "text": "He suddenly realised that I couldn't go to the house.\"", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with a subordinate clause, despite stray quote.", "agrees": true }, { "audit_id": "A089", "model": "state-spaces/mamba-130m-hf", "pair_id": "P018", "text": "At the entrance to the park, an old man wearing the same costume of a man in white and a hooded mask sat down at the front of the park with a bottle of beer in his hand.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Valid sentence with clear subject and verb.", "agrees": true }, { "audit_id": "A090", "model": "SlayerLab/pollock-mini-lm-125m", "pair_id": "P023", "text": "A few minutes later, I discovered that his hand was being pushed by a gigantic man named Skeleton.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with clear subject, verb, and object.", "agrees": true }, { "audit_id": "A091", "model": "facebook/opt-125m", "pair_id": "P073", "text": "We were looking for a good one.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with subject 'We' and verb 'were looking for a good one.'", "agrees": true }, { "audit_id": "A092", "model": "openai-community/gpt2", "pair_id": "P006", "text": "Our closest neighbours are to be found in the north, but some cities, like London, are much more diverse than those of London.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid syntax.", "agrees": true }, { "audit_id": "A093", "model": "EleutherAI/pythia-70m", "pair_id": "P010", "text": "Everyone in the room, and in between the two the three members were a little more than the two of them (with no other name) and the four of them were all one member (together with a brother).", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment with dangling conjunction and incoherent structure.", "agrees": true }, { "audit_id": "A094", "model": "EleutherAI/pythia-70m", "pair_id": "P053", "text": "Because nobody answered, it's a simple question.", "assistant_pass": true, "assistant_reason": "Well-formed complete sentence; factual or semantic oddity ignored.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with valid syntax.", "agrees": true }, { "audit_id": "A095", "model": "SlayerLab/GoLLeM-45M-PL", "pair_id": "P060", "text": "Whenever I hear that song, I got to do it.", "assistant_pass": null, "assistant_reason": "Present habitual clause with got to is ambiguous between colloquial obligation and mismatched past.", "qwen_pass": true, "qwen_verdict": "correct", "qwen_reason": "Complete sentence with a finite verb, though 'got to' is informal for 'have to'.", "agrees": null }, { "audit_id": "A096", "model": "McGill-NLP/TLM-100M", "pair_id": "P040", "text": "Several empty bottles of their favorite product is just as important to a successful event as the liquid itself.", "assistant_pass": false, "assistant_reason": "Plural several bottles conflicts with singular is.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Subject-verb agreement error: plural subject 'bottles' with singular verb 'is'.", "agrees": true }, { "audit_id": "A097", "model": "SlayerLab/slayer-scratch", "pair_id": "P071", "text": "She carefully opened of the Game of the Some the Sing of the Snowles of the Cummer.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Preposition 'of' used incorrectly after transitive verb 'opened'.", "agrees": true }, { "audit_id": "A098", "model": "SlayerLab/GoLLeM-110M-PL-v3", "pair_id": "P005", "text": "The children in the garden\" (\"Psychologia\"), a także nowe utwory (\"Wrota\", \"Fucking\", \"Pochodzenie\", \"Fucking\", \"Shield\", \"Intuited\", \"Shield\", \"Shield\", \"Shooting\") i \"The Undertown\".", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Fragment with Polish text; no main verb for the subject.", "agrees": true }, { "audit_id": "A099", "model": "EleutherAI/gpt-neo-125m", "pair_id": "P068", "text": "The message that arrived this morning.", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Noun phrase only; no verb.", "agrees": true }, { "audit_id": "A100", "model": "SlayerLab/slayer-scratch", "pair_id": "P084", "text": "He suddenly realised that's that you weimple to there on the likey?", "assistant_pass": false, "assistant_reason": "Fragment or malformed English clause.", "qwen_pass": false, "qwen_verdict": "incorrect", "qwen_reason": "Incoherent syntax; 'that's that you weimple' is ungrammatical.", "agrees": true } ] }