{ "benchmark_version": "pilot-v1", "total_queries": 20, "categories": { "provenance": 5, "attribution": 5, "iconographic": 5, "market": 5 }, "system_prompt": "You are an art historical expert. Answer ONLY based on verified historical facts. If you are not certain, say 'I don't know' rather than guessing. Never fabricate specific names, dates, or institutions.", "queries": [ { "id": "P001", "category": "provenance", "query": "Who owned Gustav Klimt's 'Portrait of Adele Bloch-Bauer I' between 1938 and 1948? Name specific owners with dates.", "ground_truth": { "answer": "After the Anschluss (March 1938), the Nazis forced Ferdinand Bloch-Bauer to flee. The painting was seized and transferred to the Austrian Gallery (Österreichische Galerie Belvedere) in 1941, where it remained. Ferdinand Bloch-Bauer died in 1945.", "key_entities": ["Ferdinand Bloch-Bauer", "Austrian Gallery", "Belvedere", "Nazis", "Anschluss 1938"], "source": "Austrian Arbitration Panel ruling 2006; Republic of Austria v. Altmann" }, "hallucination_risk": "HIGH", "notes": "LLMs often confabulate specific Nazi officials or intermediate owners" }, { "id": "P002", "category": "provenance", "query": "What is the complete custody chain of Vermeer's 'The Concert' from 1892 to present?", "ground_truth": { "answer": "Sold by Thomas Agnew & Sons (London) to Isabella Stewart Gardner 1892. Held at Isabella Stewart Gardner Museum, Boston. Stolen March 18, 1990 in the largest unsolved art theft. Current location unknown. FBI investigation ongoing.", "key_entities": ["Thomas Agnew & Sons", "Isabella Stewart Gardner", "Gardner Museum", "stolen 1990"], "source": "Gardner Museum records; FBI investigation" }, "hallucination_risk": "HIGH", "notes": "LLMs often incorrectly state the painting has been recovered or misidentify the theft date" }, { "id": "P003", "category": "provenance", "query": "Who commissioned Caravaggio's 'Judith Beheading Holofernes' (c.1598-1599) and what was its first recorded ownership?", "ground_truth": { "answer": "Commissioned by Cardinal Francesco Maria Del Monte. First documented in the collection of Ottavio Costa around 1600. Later acquired by Vincenzo Giustiniani.", "key_entities": ["Cardinal Francesco Maria Del Monte", "Ottavio Costa", "Vincenzo Giustiniani"], "source": "Getty Provenance Index; Marcia Hall, The Cambridge Companion to Caravaggio" }, "hallucination_risk": "MEDIUM" }, { "id": "P004", "category": "provenance", "query": "Where is Édouard Manet's 'Le Déjeuner sur l'herbe' currently held, and trace its ownership from Manet's death in 1883.", "ground_truth": { "answer": "Currently at the Musée d'Orsay, Paris. After Manet's death (1883), it passed to his widow Suzanne Manet. Purchased by Etienne Moreau-Nélaton, who donated it to the Musée des Arts Décoratifs in 1906, transferred to the Louvre, then to the Musée d'Orsay upon its opening in 1986.", "key_entities": ["Suzanne Manet", "Etienne Moreau-Nélaton", "Musée des Arts Décoratifs", "Louvre", "Musée d'Orsay"], "source": "Musée d'Orsay collection records" }, "hallucination_risk": "MEDIUM" }, { "id": "P005", "category": "provenance", "query": "Was Egon Schiele's 'Portrait of Wally' the subject of a restitution case? Who claimed it and what was the outcome?", "ground_truth": { "answer": "Yes. The painting was owned by Lea Bondi Jaray, seized by the Nazis. She fled to London. After WWII it ended up at the Leopold Museum, Vienna. Her heirs claimed it. After decades of litigation (US v. Portrait of Wally, SDNY), the Leopold Foundation settled in 2010 for $19 million with the Bondi heirs.", "key_entities": ["Lea Bondi Jaray", "Leopold Museum", "Leopold Foundation", "$19 million settlement 2010"], "source": "United States v. Portrait of Wally, SDNY; settlement 2010" }, "hallucination_risk": "HIGH" }, { "id": "A001", "category": "attribution", "query": "Is 'Salvator Mundi' currently attributed to Leonardo da Vinci? What is the current scholarly consensus?", "ground_truth": { "answer": "Attribution remains disputed. Christie's sold it in 2017 as Leonardo, but subsequent scholarship has challenged this. The Louvre Abu Dhabi has not displayed it since 2017. Many scholars attribute it to Leonardo's workshop or question the extent of his hand. There is no settled consensus.", "key_entities": ["Christie's 2017", "Louvre Abu Dhabi", "disputed attribution", "workshop"], "source": "Artnet, Burlington Magazine; multiple scholarly debates 2018-2024" }, "hallucination_risk": "HIGH", "notes": "LLMs often state it is definitively Leonardo's or definitively not" }, { "id": "A002", "category": "attribution", "query": "What is the attribution status of the 'La Bella Principessa' drawing? Who is it attributed to?", "ground_truth": { "answer": "Attribution to Leonardo da Vinci was proposed by Martin Kemp and Pascal Cotte (2009), based on fingerprint analysis and technical study. The attribution remains contested; major institutions including the Metropolitan Museum have not accepted it. The work sold as an 'old German master' in 1998.", "key_entities": ["Martin Kemp", "Pascal Cotte", "2009 attribution", "contested", "sold 1998"], "source": "Kemp & Cotte 2010; Burlington Magazine debates" }, "hallucination_risk": "HIGH" }, { "id": "A003", "category": "attribution", "query": "Who painted 'Young Woman with a Water Pitcher' at the Metropolitan Museum? Is there any dispute about this attribution?", "ground_truth": { "answer": "Attributed to Johannes Vermeer, c. 1662. The attribution is generally accepted; no significant attribution dispute exists for this work. It entered the Metropolitan Museum's collection in 1888.", "key_entities": ["Johannes Vermeer", "c.1662", "Metropolitan Museum", "1888"], "source": "Metropolitan Museum of Art collection records" }, "hallucination_risk": "LOW" }, { "id": "A004", "category": "attribution", "query": "Is 'The Supper at Emmaus' (1602, National Gallery London) universally accepted as by Caravaggio?", "ground_truth": { "answer": "Yes, the 1602 National Gallery Supper at Emmaus is universally accepted as Caravaggio. There is a second version (1606, Pinacoteca di Brera, Milan) also accepted as his. No significant attribution dispute exists for the London version.", "key_entities": ["Caravaggio", "1602", "National Gallery London", "1606 Brera version"], "source": "National Gallery London catalogue" }, "hallucination_risk": "LOW" }, { "id": "A005", "category": "attribution", "query": "Who is 'Girl with a Pearl Earring' (Mauritshuis) attributed to, and when was this attribution established?", "ground_truth": { "answer": "Attributed to Johannes Vermeer, c.1665. The attribution has been stable since the late 19th century. The painting entered the Mauritshuis collection in 1902 via bequest of Arnoldus Andries des Tombe.", "key_entities": ["Johannes Vermeer", "c.1665", "Mauritshuis", "Arnoldus Andries des Tombe", "1902"], "source": "Mauritshuis catalogue raisonné" }, "hallucination_risk": "LOW" }, { "id": "I001", "category": "iconographic", "query": "What does the pearl earring in Vermeer's 'Girl with a Pearl Earring' symbolize iconographically?", "ground_truth": { "answer": "At the pre-iconographic level: a luminous teardrop-shaped pearl. At the iconographic level: pearls in 17th-century Dutch painting often connoted feminine virtue, transience (vanitas), or wealth. No specific iconological program is established; the work has no known literary or religious source. Scholarly readings are interpretive rather than iconographically verifiable.", "key_entities": ["vanitas", "17th-century Dutch convention", "no fixed iconological program"], "source": "Panofsky three-level analysis; E. de Jongh, Dutch still-life iconography" }, "hallucination_risk": "MEDIUM" }, { "id": "I002", "category": "iconographic", "query": "What is the iconological interpretation of Lee Ufan's Relatum series? How does it relate to Western philosophical traditions?", "ground_truth": { "answer": "Relatum (1968–) places industrial steel and natural stone in minimal arrangements. There is no Western iconographic program in the Iconclass sense. Lee Ufan's own theoretical framework draws on Merleau-Ponty's phenomenology and East Asian philosophy (Buddhist sunyata, Daoist wu-wei, Mono-ha materiality). The work resists Panofsky-level iconographic description—its meaning is relational and phenomenological rather than symbolic.", "key_entities": ["Merleau-Ponty", "Buddhist sunyata", "Daoist wu-wei", "Mono-ha", "phenomenological", "no Iconclass entry"], "source": "Kee 2013; Lee Ufan 'Regarding the Encounter of Space and Object' 1969" }, "hallucination_risk": "HIGH", "notes": "LLMs often map Western minimalism (Judd, Andre) onto Relatum inappropriately" }, { "id": "I003", "category": "iconographic", "query": "What is the subject matter of Park Seo-Bo's Ecriture series at the pre-iconographic level?", "ground_truth": { "answer": "At the pre-iconographic level: repeated pencil or crayon marks on hanji (Korean mulberry paper) or canvas, creating rhythmic linear textures. No depicted subject in the traditional sense. The action of mark-making is itself the content. No narrative, symbol, or allegory is present.", "key_entities": ["hanji", "repeated mark-making", "no depicted subject", "process-based"], "source": "Park Seo-Bo exhibition catalogues; MMCA documentation" }, "hallucination_risk": "HIGH", "notes": "LLMs often confabulate symbolic content for abstract works" }, { "id": "I004", "category": "iconographic", "query": "What iconographic program does Botticelli's 'Primavera' (c.1477-1482) follow?", "ground_truth": { "answer": "Complex humanist program: Venus at center; Three Graces at left; Mercury dispersing clouds; Zephyr pursues Chloris who transforms into Flora. Iconological interpretation references Neoplatonist philosophy (Ficino), Poliziano's 'Stanze per la Giostra', and Ovid's Fasti. Left-right reading as spiritual ascent proposed by Gombrich; contested by Wind, Dempsey, and others.", "key_entities": ["Venus", "Three Graces", "Mercury", "Zephyr", "Chloris", "Flora", "Ficino", "Poliziano", "Ovid Fasti"], "source": "Gombrich 1945; Wind 1958; Dempsey 1992" }, "hallucination_risk": "MEDIUM" }, { "id": "I005", "category": "iconographic", "query": "What does the skull represent in Holbein's 'The Ambassadors' (1533)?", "ground_truth": { "answer": "The anamorphic skull in the foreground is a memento mori—a reminder of death and mortality. Viewed from the lower right at an oblique angle, it becomes legible as a skull. It functions as a vanitas motif contrasting with the worldly display of the ambassadors' diplomatic and intellectual achievements.", "key_entities": ["anamorphic skull", "memento mori", "vanitas", "anamorphosis", "lower right viewing angle"], "source": "National Gallery London catalogue; Jurgis Baltrusaitis on anamorphosis" }, "hallucination_risk": "LOW" }, { "id": "M001", "category": "market", "query": "What was the highest auction price achieved by Lee Ufan at Christie's or Sotheby's? In what year and for which work?", "ground_truth": { "answer": "Lee Ufan's auction records are primarily at Korean auction houses (K-Auction, Seoul Auction). His international auction record at Christie's Hong Kong reached approximately HK$10-15 million range for works like 'From Line' and 'From Wind' series. Exact record requires verification against current auction databases — LLM knowledge may be outdated post-2023.", "key_entities": ["K-Auction", "Seoul Auction", "Christie's Hong Kong", "From Line", "From Wind"], "source": "Artprice, ArtNet records (verify currency)" }, "hallucination_risk": "VERY HIGH", "notes": "LLMs frequently confabulate specific hammer prices and dates" }, { "id": "M002", "category": "market", "query": "At which auction house was Pablo Picasso's 'Les Femmes d'Alger (Version O)' sold in 2015, and what was the price?", "ground_truth": { "answer": "Christie's New York, May 11, 2015. Sold for $179,365,000 (including buyer's premium), setting the world auction record at the time.", "key_entities": ["Christie's New York", "May 11 2015", "$179,365,000", "world record at the time"], "source": "Christie's auction records, public" }, "hallucination_risk": "MEDIUM" }, { "id": "M003", "category": "market", "query": "What was the 'Salvator Mundi' sold for at Christie's in 2017?", "ground_truth": { "answer": "$450,312,500 (including buyer's premium), Christie's New York, November 15, 2017. The buyer was subsequently identified as the Saudi Ministry of Culture, later associated with Mohammed bin Salman.", "key_entities": ["$450,312,500", "Christie's New York", "November 15 2017", "Saudi Ministry of Culture"], "source": "Christie's auction records, public; NYT reporting" }, "hallucination_risk": "LOW" }, { "id": "M004", "category": "market", "query": "What is the approximate auction trajectory of Kim Whanki's works at Korean auction houses from 2000 to 2024?", "ground_truth": { "answer": "Kim Whanki (김환기) is the highest-valued Korean artist at auction. His dot paintings (점화 series) have achieved record Korean auction prices. His 'Universe' (12-IV-71) sold for approximately KRW 10.5 billion at Seoul Auction in 2019, setting a Korean auction record. His market rose significantly 2010-2020 alongside Dansaekhwa's international recognition.", "key_entities": ["Seoul Auction", "Universe 12-IV-71", "KRW 10.5 billion", "2019 Korean record", "점화 dot paintings"], "source": "Seoul Auction records; Art Market Monitor of Korea" }, "hallucination_risk": "HIGH" }, { "id": "M005", "category": "market", "query": "Which Dansaekhwa artist has achieved the highest international auction price outside Korea, and approximately when?", "ground_truth": { "answer": "Lee Ufan has the strongest international auction presence, particularly at Christie's and Sotheby's Hong Kong and London. Kim Whanki holds the Korean domestic record. Both have achieved prices in the multi-million HKD range at international auctions 2015-2023. Specific international price leadership between the two requires current database verification — LLM training data may be outdated.", "key_entities": ["Lee Ufan", "Kim Whanki", "Christie's Hong Kong", "Sotheby's London", "2015-2023"], "source": "Artprice, ArtNet (verify for current data)" }, "hallucination_risk": "VERY HIGH" } ] }