{"paper":{"id":"2609.00038.pdf","title":"trajectory-judge: What Outcome-Only LLM Judges Miss on Agent Trajectories","authors":["Hadi Mohammadi"],"arxiv_id":"","url":"","source_kind":"pdf","ir_version":"1.0"},"generated_at":"2026-09-14T04:01:35.245218+00:00","counts":{"error":0,"warning":0,"info":0},"findings":[],"top_findings":[],"ir_summary":{"claims":22,"contributions":4,"method_components":8,"experiments":7,"baselines":4,"datasets":0,"metrics":7,"tables":8,"figures":3,"citations":35,"equations":0,"quantitative_results":604,"has_repository":false},"ir_health":{"observations":[{"code":"compile-warning","severity":"note","message":"references: the reference list carries no entry numbers; citation keys are enumeration order [1]..[N], not labels printed in the paper, so a key may not match the [N] used in the body","detail":""},{"code":"compile-warning","severity":"note","message":"semantic extraction: dropped 7 items whose quote was not found in the paper","detail":""}],"errors":0,"warnings":0,"trustworthy":true},"diagnostics":{"source":"/home/oceann/workspace/research-linter/eval/papers/2609.00038.pdf","rules_run":["RL001","RL002","RL003","RL004","RL005"],"rules_skipped":[],"rule_errors":{},"rule_findings":{"RL001":0,"RL002":0,"RL003":0,"RL004":0,"RL005":0},"rule_seconds":{"RL001":0.01,"RL002":0.0,"RL003":2.32,"RL004":0.0,"RL005":0.07},"dropped_no_evidence":0,"demoted_low_confidence":0,"ir_health":{"observations":[{"code":"compile-warning","severity":"note","message":"references: the reference list carries no entry numbers; citation keys are enumeration order [1]..[N], not labels printed in the paper, so a key may not match the [N] used in the body","detail":""},{"code":"compile-warning","severity":"note","message":"semantic extraction: dropped 7 items whose quote was not found in the paper","detail":""}],"errors":0,"warnings":0,"trustworthy":true},"notes":["RL001 funnel arithmetic: sentences_with_from=4 endpoint_pairs=1 claims_compared=1 emitted=0 [dropped: not-a-measurement=0]","RL001 funnel delta-column: structured_tables=8 with_exactly_one_delta_column=1 rows_checked=0 emitted=0","RL001 funnel cross-location: quant=604 prose=281 prose_with_metric=184 prose_with_dataset=0 prose_with_metric_and_dataset=0 distinct_buckets=0 emitted=0 [dropped: not-a-measurement=0 not-in-quote=30]","RL001 funnel identifier-drift: settings_checked=2 sections_stating_one_value=1 emitted=0","RL002: 0 checkable cited claims (0 in-text markers, 0 not linked to a bibliography entry, 0 dropped as non-citations, 0 jointly attributed to several works)","RL002 citation entailment: attempted 0 of 0 ranked cited claims (0 in-text markers from 35 bibliography entries); 0 cited works could not be identified, 0 had no retrievable text, 0 model quotes failed verbatim verification, 0 were topically unrelated to the retrieved text.","RL003: 2 queries -> 116 unique works (arXiv search 66, category listings 0, scholarly graph 50 [crossref 50], 0 graph works matched to an arXiv id); source errors during retrieval: arXiv 0, graph 0","RL003: parsed the reference list of 6/10 comparable papers (4 unreadable) — that is the PeerUsage denominator","RL003: enriched 0 of 0 candidates with their own abstract (0 unavailable, 0 over budget); 0 dropped as published after this paper","RL003: scholarly graph for this run = crossref","RL003 funnel (446 candidates): cited_only_from_related_work -147, is_the_paper_itself -1, peer_usage_below_quorum -55, survey_or_artifact -50, title_names_no_method -193 || kept: citation_context_unknown +36 || survived-cheap-gates 0 -> with-abstract 0 -> comparable 0 -> above-threshold-0.65 0","RL003: 0 comparable candidates all scored below 0.65 — nothing to report for this paper","RL004: no experiment pairs a metric with a dataset — this paper runs no task-level evaluation, so a missing-ablation finding would not be actionable; emitting nothing","RL005: https://github.com/mohammadi-hadi/trajectory-judge — read 33 files, 0 canonical params, 1 paper claims, 0 mismatches"],"missing_core_rules":[],"seconds":10.61,"llm":{"calls":0,"cache_hits":1,"errors":0,"prompt_chars":0},"sources":{"arxiv":{"requests":1,"errors":0,"cache_hits":24,"available":1},"openalex":{"requests":1,"errors":0,"cache_hits":0,"available":0,"unavailable_reason":"OpenAlex free daily budget exhausted (resets midnight UTC)"},"crossref":{"requests":0,"errors":0,"cache_hits":2,"available":1},"s2":{"requests":0,"errors":0,"cache_hits":0,"available":1},"short_circuited":[]},"cache":{"hits":27,"misses":2}},"compile_warnings":["references: the reference list carries no entry numbers; citation keys are enumeration order [1]..[N], not labels printed in the paper, so a key may not match the [N] used in the body","semantic extraction: dropped 7 items whose quote was not found in the paper"]}