📦 deps(skills): sync thirdparty skills
This commit is contained in:
ci[bot]
+36
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"kind": "webfonts#webfontList",
|
||||
"items": [
|
||||
{
|
||||
"family": "Zeta Serif",
|
||||
"variants": ["700italic", "regular", "700"],
|
||||
"subsets": ["latin-ext", "latin"],
|
||||
"version": "v2",
|
||||
"lastModified": "2025-06-02",
|
||||
"files": {
|
||||
"regular": "https://fonts.gstatic.com/zeta-regular.ttf",
|
||||
"700": "https://fonts.gstatic.com/zeta-700.ttf",
|
||||
"700italic": "https://fonts.gstatic.com/zeta-700-italic.ttf"
|
||||
},
|
||||
"category": "serif",
|
||||
"kind": "webfonts#webfont"
|
||||
},
|
||||
{
|
||||
"family": "Alpha Sans",
|
||||
"variants": ["regular", "italic", "500"],
|
||||
"subsets": ["vietnamese", "latin", "latin"],
|
||||
"version": "v4",
|
||||
"lastModified": "2025-01-03",
|
||||
"files": {
|
||||
"regular": "https://fonts.gstatic.com/alpha-regular.ttf",
|
||||
"italic": "https://fonts.gstatic.com/alpha-italic.ttf",
|
||||
"500": "https://fonts.gstatic.com/alpha-500.ttf"
|
||||
},
|
||||
"category": "sans-serif",
|
||||
"kind": "webfonts#webfont",
|
||||
"axes": [
|
||||
{"tag": "wght", "start": 100, "end": 900}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"axisRegistry": [],
|
||||
"familyMetadataList": [
|
||||
{
|
||||
"family": "Zeta Serif", "displayName": null, "category": "Serif", "stroke": "Serif",
|
||||
"classifications": ["Display"], "size": 1000, "subsets": ["menu", "latin-ext", "latin"],
|
||||
"fonts": {
|
||||
"400": {"thickness": 5, "slant": 1, "width": 7, "lineHeight": 1.2},
|
||||
"700": {"thickness": 7, "slant": 1, "width": 7, "lineHeight": 1.2},
|
||||
"700i": {"thickness": 7, "slant": 4, "width": 7, "lineHeight": 1.2}
|
||||
}, "axes": [], "designers": ["Zeta Studio"],
|
||||
"lastModified": "2025-06-02", "dateAdded": "2018-04-09", "popularity": 90,
|
||||
"trending": 11, "defaultSort": 9, "androidFragment": null, "isNoto": false,
|
||||
"colorCapabilities": [], "primaryScript": "", "primaryLanguage": "", "isOpenSource": true,
|
||||
"isBrandFont": false, "languages": []
|
||||
},
|
||||
{
|
||||
"family": "Alpha Sans", "displayName": null, "category": "Sans Serif", "stroke": "Sans Serif",
|
||||
"classifications": ["Geometric"], "size": 2000, "subsets": ["menu", "vietnamese", "latin"],
|
||||
"fonts": {
|
||||
"400": {"thickness": 5, "slant": 1, "width": 7, "lineHeight": 1.2},
|
||||
"400i": {"thickness": 5, "slant": 4, "width": 7, "lineHeight": 1.2},
|
||||
"500": {"thickness": 6, "slant": 1, "width": 7, "lineHeight": 1.2}
|
||||
},
|
||||
"axes": [{"tag": "wght", "min": 100.0, "max": 900.0, "defaultValue": 400.0}],
|
||||
"designers": ["Ada Type", "Binh Fonts"], "lastModified": "2025-01-03",
|
||||
"dateAdded": "2020-02-20", "popularity": 42, "trending": 7, "defaultSort": 4,
|
||||
"androidFragment": null, "isNoto": false, "colorCapabilities": [], "primaryScript": "",
|
||||
"primaryLanguage": "", "isOpenSource": true, "isBrandFont": false, "languages": []
|
||||
}
|
||||
],
|
||||
"promotedScript": []
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
Family,Category,Stroke,Classifications,Keywords,Styles,Variable Axes,Subsets,Designers,Popularity Rank,Trending Rank,Is Noto,Date Added,Last Modified,Google Fonts URL
|
||||
Alpha Sans,Sans Serif,Sans Serif,Geometric,reviewed clean keywords,400,,,Old Designer,42,7,No,2020-02-20,2024-01-01,https://fonts.google.com/specimen/Alpha+Sans
|
||||
Zeta Serif,Serif,Serif,Display,reviewed editorial keywords,400,,,Old Studio,90,11,No,2018-04-09,2024-01-01,https://fonts.google.com/specimen/Zeta+Serif
|
||||
|
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"families": [
|
||||
{
|
||||
"name": "Zeta Serif",
|
||||
"designer": "Zeta Studio",
|
||||
"license": "APACHE2",
|
||||
"date_added": "2018-04-09"
|
||||
},
|
||||
{
|
||||
"name": "Alpha Sans",
|
||||
"designer": ["Ada Type", "Binh Fonts"],
|
||||
"license": "OFL",
|
||||
"date_added": "2020-02-20"
|
||||
}
|
||||
],
|
||||
"excludedFamilies": []
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"families": {
|
||||
"Alpha Sans": {
|
||||
"Keywords": "approved override keywords"
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
No,Category,Icon Name,Keywords,Library,Import Code,Usage,Best For,Style,Semantic Role,Allowed Contexts
|
||||
1,Navigation,arrow-left,back,Phosphor,import { ArrowLeft } from '@phosphor-icons/react',Example,Back,Outline,interactive,interactive
|
||||
2,Nature,acorn,nut,Phosphor,import { Acorn } from '@phosphor-icons/react',Example,Acorn,Outline,meaningful,meaningful
|
||||
3,Guideline,example,guidance,Heroicons,import { BeakerIcon } from '@heroicons/react/24/outline',Example,Example,Outline,guideline,meaningful
|
||||
|
+23
@@ -0,0 +1,23 @@
|
||||
[
|
||||
{
|
||||
"name": "acorn",
|
||||
"pascal_name": "Acorn",
|
||||
"codepoint": 62002,
|
||||
"categories": ["animals", "nature"],
|
||||
"figma_category": "weather & nature",
|
||||
"tags": ["savings", "food"],
|
||||
"published_in": 1.2,
|
||||
"updated_in": 2.0
|
||||
},
|
||||
{
|
||||
"name": "arrow-left",
|
||||
"pascal_name": "ArrowLeft",
|
||||
"alias": {"name": "back-arrow", "pascal_name": "BackArrow"},
|
||||
"codepoint": 62000,
|
||||
"categories": ["arrows", "navigation", "arrows"],
|
||||
"figma_category": "arrows",
|
||||
"tags": ["previous", "back"],
|
||||
"published_in": 1.0,
|
||||
"updated_in": 2.1
|
||||
}
|
||||
]
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "@phosphor-icons/core",
|
||||
"version": "2.1.1",
|
||||
"license": "MIT",
|
||||
"exports": {
|
||||
"./thin/*.svg": "./assets/thin/*.svg",
|
||||
"./light/*.svg": "./assets/light/*.svg",
|
||||
"./regular/*.svg": "./assets/regular/*.svg",
|
||||
"./bold/*.svg": "./assets/bold/*.svg",
|
||||
"./fill/*.svg": "./assets/fill/*.svg",
|
||||
"./duotone/*.svg": "./assets/duotone/*.svg"
|
||||
}
|
||||
}
|
||||
Vendored
+4
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"client": ["Acorn", "ArrowLeft", "IconContext"],
|
||||
"ssr": ["Acorn", "ArrowLeft"]
|
||||
}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"name": "@phosphor-icons/react",
|
||||
"version": "2.1.10",
|
||||
"license": "MIT"
|
||||
}
|
||||
+3366
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"fixtureRevision": "97eb2a2",
|
||||
"gradeSemantics": {
|
||||
"2": "Directly answers the query and is an expected best result.",
|
||||
"1": "Useful and acceptable, but less specific than a grade-2 result.",
|
||||
"0": "Not relevant. Every returned row not listed in judgments is implicitly grade 0."
|
||||
},
|
||||
"globalNegativeApplicability": {
|
||||
"tag": "hard-negative",
|
||||
"domains": ["style", "color", "chart", "landing", "product", "ux", "typography", "icons", "gsap", "react", "web", "google-fonts"],
|
||||
"stacks": ["react", "nextjs", "vue", "svelte", "astro", "swiftui", "react-native", "flutter", "nuxtjs", "nuxt-ui", "html-tailwind", "shadcn", "jetpack-compose", "threejs", "angular", "laravel", "javafx", "wpf", "winui", "avalonia", "uno", "uwp"],
|
||||
"interpretation": "Run every hard-negative case against all registered domains and stacks. Any returned row is an implicit grade-0 false positive; aggregate reporting may cap this cross-product separately."
|
||||
},
|
||||
"cases": [
|
||||
{"id":"domain-style-glassmorphism","split":"calibration","mode":"domain","query":"frosted glass transparent blurred layered UI","domain":"style","judgments":[{"identity":{"Style Category":"Glassmorphism"},"grade":2}],"tags":["exact-intent","domain-positive"],"notes":"Canonical visual treatment, expressed through its descriptive attributes."},
|
||||
{"id":"domain-style-accessible-paraphrase","split":"held_out","mode":"domain","query":"inclusive high contrast interface with keyboard nav and screen readers","domain":"style","judgments":[{"identity":{"Style Category":"Accessible & Ethical"},"grade":2},{"identity":{"Style Category":"Inclusive Design"},"grade":1}],"tags":["paraphrase","domain-positive"],"notes":"A style request, not an individual UX rule."},
|
||||
{"id":"domain-color-spa","split":"calibration","mode":"domain","query":"beauty spa wellness soft pink lavender palette","domain":"color","judgments":[{"identity":{"Product Type":"Beauty/Spa/Wellness Service"},"grade":2}],"tags":["product-palette","domain-positive"],"notes":"Uses the public Product Type as the stable palette identity."},
|
||||
{"id":"domain-color-cyber-typo","split":"held_out","mode":"domain","query":"cybersecurty threat dashboard matrix green dark pallete","domain":"color","judgments":[{"identity":{"Product Type":"Cybersecurity Platform"},"grade":2}],"tags":["typo","domain-positive"],"notes":"Deliberately misspells cybersecurity and palette."},
|
||||
{"id":"domain-chart-time-series","split":"calibration","mode":"domain","query":"trend over time growth timeline line chart","domain":"chart","judgments":[{"identity":{"Data Type":"Trend Over Time"},"grade":2},{"identity":{"Data Type":"Time-Series Forecast"},"grade":1}],"tags":["exact-intent","domain-positive"],"notes":"Forecasting is acceptable only as a secondary interpretation."},
|
||||
{"id":"domain-chart-correlation-paraphrase","split":"held_out","mode":"domain","query":"show whether two measures move together using dots and bubbles","domain":"chart","judgments":[{"identity":{"Data Type":"Correlation / Distribution"},"grade":2}],"tags":["paraphrase","domain-positive"],"notes":"Avoids the canonical words scatter and correlation."},
|
||||
{"id":"domain-landing-pricing","split":"calibration","mode":"domain","query":"pricing plans tiers comparison landing CTA","domain":"landing","judgments":[{"identity":{"Pattern Name":"Pricing Page + CTA"},"grade":2},{"identity":{"Pattern Name":"Pricing-Focused Landing"},"grade":2},{"identity":{"Pattern Name":"Comparison Table + CTA"},"grade":1}],"tags":["multi-acceptable","domain-positive"],"notes":"Both dedicated pricing patterns are equally valid."},
|
||||
{"id":"domain-landing-enterprise-typo","split":"held_out","mode":"domain","query":"enterprise credibilty trust authority conversion page","domain":"landing","judgments":[{"identity":{"Pattern Name":"Trust & Authority + Conversion"},"grade":2},{"identity":{"Pattern Name":"Enterprise Gateway"},"grade":1}],"tags":["typo","domain-positive"],"notes":"Misspells credibility while preserving the enterprise intent."},
|
||||
{"id":"domain-product-spa","split":"calibration","mode":"domain","query":"salon massage skincare booking and wellness service","domain":"product","judgments":[{"identity":{"Product Type":"Beauty/Spa/Wellness Service"},"grade":2},{"identity":{"Product Type":"Booking & Appointment App"},"grade":1}],"tags":["multi-acceptable","domain-positive"],"notes":"The industry is primary; booking is a supporting capability."},
|
||||
{"id":"domain-product-security-paraphrase","split":"held_out","mode":"domain","query":"platform for monitoring digital threats and protecting systems","domain":"product","judgments":[{"identity":{"Product Type":"Cybersecurity Platform"},"grade":2}],"tags":["paraphrase","domain-positive"],"notes":"Avoids the exact keyword cyber."},
|
||||
{"id":"domain-ux-keyboard-focus","split":"calibration","mode":"domain","query":"visible focus and complete keyboard navigation for web users","domain":"ux","judgments":[{"identity":{"Category":"Accessibility","Issue":"Keyboard Navigation","Platform":"Web"},"grade":2},{"identity":{"Category":"Interaction","Issue":"Focus States","Platform":"All"},"grade":2}],"tags":["multi-acceptable","domain-positive"],"notes":"Both navigation coverage and visible focus are explicitly requested."},
|
||||
{"id":"domain-ux-motion-paraphrase","split":"held_out","mode":"domain","query":"stop animations making people sick and honor their motion preference","domain":"ux","judgments":[{"identity":{"Category":"Animation","Issue":"Reduced Motion","Platform":"All"},"grade":2},{"identity":{"Category":"Animation","Issue":"Excessive Motion","Platform":"All"},"grade":1}],"tags":["paraphrase","accessibility","domain-positive"],"notes":"Preference support is stronger than generic excessive-motion advice."},
|
||||
{"id":"domain-typography-luxury","split":"calibration","mode":"domain","query":"elegant luxury serif heading with clean readable body font","domain":"typography","judgments":[{"identity":{"Font Pairing Name":"Classic Elegant"},"grade":2},{"identity":{"Font Pairing Name":"Luxury Serif"},"grade":2},{"identity":{"Font Pairing Name":"Luxury Minimalist"},"grade":1}],"tags":["multi-acceptable","domain-positive"],"notes":"Several curated pairs directly satisfy this intentionally broad mood."},
|
||||
{"id":"domain-typography-accessible-typo","split":"held_out","mode":"domain","query":"dyslexia frendly hyperlegible inclusive type pairing","domain":"typography","judgments":[{"identity":{"Font Pairing Name":"Accessibility First"},"grade":2},{"identity":{"Font Pairing Name":"Academic/Research"},"grade":1}],"tags":["typo","accessibility","domain-positive"],"notes":"Misspells friendly; the all-Atkinson pairing is the direct answer."},
|
||||
{"id":"domain-icons-search","split":"calibration","mode":"domain","query":"find lookup search icon for a query field","domain":"icons","judgments":[{"identity":{"Category":"Action","Icon Name":"magnifying-glass","Library":"Phosphor"},"grade":2},{"identity":{"Category":"Action","Icon Name":"funnel","Library":"Phosphor"},"grade":1}],"tags":["exact-intent","domain-positive"],"notes":"Filter is useful but not equivalent to search."},
|
||||
{"id":"domain-icons-warning-typo","split":"held_out","mode":"domain","query":"warnng caution danger status symbol","domain":"icons","judgments":[{"identity":{"Category":"Status","Icon Name":"warning","Library":"Phosphor"},"grade":2},{"identity":{"Category":"Status","Icon Name":"warning-circle","Library":"Phosphor"},"grade":1}],"tags":["typo","domain-positive"],"notes":"Misspells warning and allows the circled variant as secondary."},
|
||||
{"id":"domain-gsap-scroll-reveal","split":"calibration","mode":"domain","query":"GSAP reveal elements when they enter the viewport on scroll","domain":"gsap","judgments":[{"identity":{"Category":"Scroll Reveal","Intensity Tier":"Subtle","Trigger":"scroll (viewport enter)"},"grade":2},{"identity":{"Category":"Scroll Reveal","Intensity Tier":"Standard","Trigger":"scroll (viewport enter)"},"grade":2},{"identity":{"Category":"Scroll Reveal","Intensity Tier":"Complex","Trigger":"scroll (continuous scrub)"},"grade":1}],"tags":["multi-acceptable","domain-positive"],"notes":"Viewport-entry variants are direct; continuous scrub is related but stronger."},
|
||||
{"id":"domain-gsap-skeleton-typo","split":"held_out","mode":"domain","query":"skeletn shimmer loader while async content waits","domain":"gsap","judgments":[{"identity":{"Category":"Loading / Skeleton","Intensity Tier":"Subtle","Trigger":"on mount / async wait"},"grade":2},{"identity":{"Category":"Loading / Skeleton","Intensity Tier":"Standard","Trigger":"on mount / async wait"},"grade":1}],"tags":["typo","domain-positive"],"notes":"Misspells skeleton; subtle shimmer is the most exact row."},
|
||||
{"id":"domain-react-parallel-promises","split":"calibration","mode":"domain","query":"React Promise.all parallel concurrent requests instead of waterfall","domain":"react","judgments":[{"identity":{"Category":"Async Waterfall","Issue":"Promise.all Parallel","Platform":"React/Next.js"},"grade":2},{"identity":{"Category":"Async Waterfall","Issue":"Dependency Parallelization","Platform":"React/Next.js"},"grade":1}],"tags":["exact-intent","domain-positive"],"notes":"The first row exactly names Promise.all."},
|
||||
{"id":"domain-react-dynamic-import-paraphrase","split":"held_out","mode":"domain","query":"load a heavy JavaScript chunk only when the component is needed","domain":"react","judgments":[{"identity":{"Category":"Bundle Size","Issue":"Dynamic Imports","Platform":"React/Next.js"},"grade":2},{"identity":{"Category":"Bundle Size","Issue":"Conditional Loading","Platform":"React/Next.js"},"grade":1}],"tags":["paraphrase","domain-positive"],"notes":"Avoids the canonical term lazy in the main clause."},
|
||||
{"id":"domain-web-icon-label","split":"calibration","mode":"domain","query":"accessible name for an icon-only mobile button","domain":"web","judgments":[{"identity":{"Category":"Accessibility","Issue":"Icon Button Labels","Platform":"iOS/Android/React Native"},"grade":2}],"tags":["accessibility","domain-positive"],"notes":"App-interface guidance uses the public web domain key."},
|
||||
{"id":"domain-web-virtual-list-paraphrase","split":"held_out","mode":"domain","query":"keep a very long mobile list smooth without rendering every row","domain":"web","judgments":[{"identity":{"Category":"Performance","Issue":"Virtualize Long Lists","Platform":"iOS/Android/React Native"},"grade":2}],"tags":["paraphrase","domain-positive"],"notes":"Describes virtualization without naming FlatList."},
|
||||
{"id":"domain-google-fonts-inter","split":"calibration","mode":"domain","query":"Inter variable sans serif Google font family","domain":"google-fonts","judgments":[{"identity":{"Family":"Inter"},"grade":2},{"identity":{"Family":"Inter Tight"},"grade":1}],"tags":["exact-entity","domain-positive"],"notes":"Exact family is preferred over its related Tight family."},
|
||||
{"id":"domain-google-fonts-atkinson-paraphrase","split":"held_out","mode":"domain","query":"hyperlegible accessible sans typeface for easier reading","domain":"google-fonts","judgments":[{"identity":{"Family":"Atkinson Hyperlegible"},"grade":2},{"identity":{"Family":"Atkinson Hyperlegible Next"},"grade":1}],"tags":["paraphrase","accessibility","domain-positive"],"notes":"The established family is locked; its newer related family is acceptable."},
|
||||
|
||||
{"id":"stack-angular-standalone","split":"calibration","mode":"stack","query":"standalone Angular components for a new project","stack":"angular","judgments":[{"identity":{"Category":"Components","Guideline":"Use standalone components"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Modern Angular component architecture."},
|
||||
{"id":"stack-angular-signals-paraphrase","split":"held_out","mode":"stack","query":"reactive local state with Angular signal primitives","stack":"angular","judgments":[{"identity":{"Category":"Components","Guideline":"Use signals for state"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Held out wording for signals."},
|
||||
{"id":"stack-astro-islands","split":"calibration","mode":"stack","query":"Astro islands architecture interactive components","stack":"astro","judgments":[{"identity":{"Category":"Architecture","Guideline":"Use Islands Architecture"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical Astro architecture."},
|
||||
{"id":"stack-astro-zero-js-paraphrase","split":"held_out","mode":"stack","query":"ship no browser JavaScript unless interaction requires it","stack":"astro","judgments":[{"identity":{"Category":"Architecture","Guideline":"Default to zero JS"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes the zero-JS default."},
|
||||
{"id":"stack-avalonia-namespace","split":"calibration","mode":"stack","query":"Avalonia XAML namespace declaration","stack":"avalonia","judgments":[{"identity":{"Category":"XAML","Guideline":"Use Avalonia XAML namespace"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Exact Avalonia XAML setup concern."},
|
||||
{"id":"stack-avalonia-compiled-binding","split":"held_out","mode":"stack","query":"type checked compiled binding with x DataType","stack":"avalonia","judgments":[{"identity":{"Category":"XAML","Guideline":"Use compiled bindings with x:DataType"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Punctuation-free paraphrase of x:DataType."},
|
||||
{"id":"stack-flutter-stateless","split":"calibration","mode":"stack","query":"prefer StatelessWidget when Flutter UI has no mutable state","stack":"flutter","judgments":[{"identity":{"Category":"Widgets","Guideline":"Use StatelessWidget when possible"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Direct widget choice."},
|
||||
{"id":"stack-flutter-const-paraphrase","split":"held_out","mode":"stack","query":"reduce Flutter rebuild cost with immutable compile-time widgets","stack":"flutter","judgments":[{"identity":{"Category":"Widgets","Guideline":"Use const constructors"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes const constructor value without copying the title."},
|
||||
{"id":"stack-html-tailwind-z-index","split":"calibration","mode":"stack","query":"Tailwind z-index utility scale for layered UI","stack":"html-tailwind","judgments":[{"identity":{"Category":"Z-Index","Guideline":"Use Tailwind z-* scale"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Specific utility-scale guidance."},
|
||||
{"id":"stack-html-tailwind-hover","split":"held_out","mode":"stack","query":"smooth transition when a Tailwind element is hovered","stack":"html-tailwind","judgments":[{"identity":{"Category":"Animation","Guideline":"Hover transitions"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Natural language hover-motion request."},
|
||||
{"id":"stack-javafx-application","split":"calibration","mode":"stack","query":"launch JavaFX UI from an Application subclass","stack":"javafx","judgments":[{"identity":{"Category":"Application","Guideline":"Start UI from Application subclass"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical JavaFX application lifecycle."},
|
||||
{"id":"stack-javafx-threading-paraphrase","split":"held_out","mode":"stack","query":"prevent slow background work from freezing the FX application thread","stack":"javafx","judgments":[{"identity":{"Category":"Threading","Guideline":"Keep work off the FX Application Thread"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes the failure mode rather than repeating the title."},
|
||||
{"id":"stack-compose-pure-ui","split":"calibration","mode":"stack","query":"pure Jetpack Compose UI composables","stack":"jetpack-compose","judgments":[{"identity":{"Category":"Composable","Guideline":"Pure UI composables"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Direct Compose architecture query."},
|
||||
{"id":"stack-compose-single-source","split":"held_out","mode":"stack","query":"one authoritative owner for Compose screen state","stack":"jetpack-compose","judgments":[{"identity":{"Category":"State","Guideline":"Single source of truth"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Paraphrases state ownership."},
|
||||
{"id":"stack-laravel-blade-component","split":"calibration","mode":"stack","query":"reusable Laravel Blade UI components","stack":"laravel","judgments":[{"identity":{"Category":"Blade Templates","Guideline":"Use Blade components for reusable UI"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Direct Blade reuse guidance."},
|
||||
{"id":"stack-laravel-props","split":"held_out","mode":"stack","query":"declare typed inputs for a Blade component using props","stack":"laravel","judgments":[{"identity":{"Category":"Blade Templates","Guideline":"Use @props for component type-safety"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Natural wording around @props."},
|
||||
{"id":"stack-nextjs-app-router","split":"calibration","mode":"stack","query":"Next.js App Router for a new project","stack":"nextjs","judgments":[{"identity":{"Category":"Routing","Guideline":"Use App Router for new projects"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Current routing architecture."},
|
||||
{"id":"stack-nextjs-server-components","split":"held_out","mode":"stack","query":"render on the server by default and opt into client boundaries","stack":"nextjs","judgments":[{"identity":{"Category":"Rendering","Guideline":"Use Server Components by default"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Held-out RSC wording."},
|
||||
{"id":"stack-nuxt-ui-module","split":"calibration","mode":"stack","query":"install the Nuxt UI module","stack":"nuxt-ui","judgments":[{"identity":{"Category":"Installation","Guideline":"Add Nuxt UI module"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Basic Nuxt UI setup."},
|
||||
{"id":"stack-nuxt-ui-semantic-color","split":"held_out","mode":"stack","query":"style Nuxt UI components through meaning-based color props","stack":"nuxt-ui","judgments":[{"identity":{"Category":"Components","Guideline":"Use semantic color props"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Avoids copying semantic exactly except its concept."},
|
||||
{"id":"stack-nuxtjs-file-routing","split":"calibration","mode":"stack","query":"Nuxt file-based page routing","stack":"nuxtjs","judgments":[{"identity":{"Category":"Routing","Guideline":"Use file-based routing"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical Nuxt routing convention."},
|
||||
{"id":"stack-nuxtjs-ssr","split":"held_out","mode":"stack","query":"server render Nuxt pages unless a client-only boundary is necessary","stack":"nuxtjs","judgments":[{"identity":{"Category":"Rendering","Guideline":"Use SSR by default"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes the SSR default."},
|
||||
{"id":"stack-react-native-functional","split":"calibration","mode":"stack","query":"functional React Native components","stack":"react-native","judgments":[{"identity":{"Category":"Components","Guideline":"Use functional components"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Direct component style."},
|
||||
{"id":"stack-react-native-stylesheet","split":"held_out","mode":"stack","query":"define reusable native styles outside render instead of inline objects","stack":"react-native","judgments":[{"identity":{"Category":"Styling","Guideline":"Use StyleSheet.create"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes StyleSheet.create without naming it."},
|
||||
{"id":"stack-react-usestate","split":"calibration","mode":"stack","query":"React useState for component local state","stack":"react","judgments":[{"identity":{"Category":"State","Guideline":"Use useState for local state"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical local state hook."},
|
||||
{"id":"stack-react-effect-cleanup","split":"held_out","mode":"stack","query":"remove subscriptions and timers when a React effect unmounts","stack":"react","judgments":[{"identity":{"Category":"Effects","Guideline":"Clean up effects"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes cleanup behavior."},
|
||||
{"id":"stack-shadcn-cli","split":"calibration","mode":"stack","query":"install shadcn components with the CLI","stack":"shadcn","judgments":[{"identity":{"Category":"Setup","Guideline":"Use CLI for installation"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical setup path."},
|
||||
{"id":"stack-shadcn-css-vars","split":"held_out","mode":"stack","query":"theme shadcn semantic colors through custom properties","stack":"shadcn","judgments":[{"identity":{"Category":"Theming","Guideline":"Use CSS variables for colors"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Uses custom properties as a synonym for CSS variables."},
|
||||
{"id":"stack-svelte-state","split":"calibration","mode":"stack","query":"Svelte 5 $state rune for reactive state","stack":"svelte","judgments":[{"identity":{"Category":"Reactivity","Guideline":"Use $state in Svelte 5"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Specific Svelte 5 primitive."},
|
||||
{"id":"stack-svelte-effect","split":"held_out","mode":"stack","query":"run a Svelte 5 side effect when reactive dependencies change","stack":"svelte","judgments":[{"identity":{"Category":"Reactivity","Guideline":"Use $effect for side effects"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Behavioral wording for $effect."},
|
||||
{"id":"stack-swiftui-state","split":"calibration","mode":"stack","query":"SwiftUI @State for view-local value state","stack":"swiftui","judgments":[{"identity":{"Category":"State","Guideline":"Use @State for local state"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical property wrapper choice."},
|
||||
{"id":"stack-swiftui-navigation","split":"held_out","mode":"stack","query":"modern value-driven iOS navigation container replacing NavigationView","stack":"swiftui","judgments":[{"identity":{"Category":"Navigation","Guideline":"Use NavigationStack or NavigationSplitView (iOS 16+)"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes NavigationStack migration intent."},
|
||||
{"id":"stack-threejs-orbitcontrols","split":"calibration","mode":"stack","query":"Three.js OrbitControls must be imported separately","stack":"threejs","judgments":[{"identity":{"Category":"Setup","Guideline":"Import OrbitControls from Three.js Addons"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Common setup failure."},
|
||||
{"id":"stack-threejs-pixel-ratio","split":"held_out","mode":"stack","query":"avoid excessive GPU work on retina screens by limiting renderer DPR","stack":"threejs","judgments":[{"identity":{"Category":"Setup","Guideline":"Pixel Ratio Cap at 2"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Uses DPR and performance wording."},
|
||||
{"id":"stack-uno-winui-xaml","split":"calibration","mode":"stack","query":"Uno Platform WinUI XAML API surface","stack":"uno","judgments":[{"identity":{"Category":"XAML","Guideline":"Use WinUI XAML API surface"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical cross-platform XAML surface."},
|
||||
{"id":"stack-uno-package-paraphrase","split":"held_out","mode":"stack","query":"choose the modern Uno WinUI package instead of the legacy Uno UI package","stack":"uno","judgments":[{"identity":{"Category":"XAML","Guideline":"Use Uno.WinUI not Uno.UI for new projects"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Natural package-selection wording."},
|
||||
{"id":"stack-uwp-xbind","split":"calibration","mode":"stack","query":"UWP compiled x:Bind data binding","stack":"uwp","judgments":[{"identity":{"Category":"XAML","Guideline":"Use x:Bind for compiled bindings"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical compiled binding guidance."},
|
||||
{"id":"stack-uwp-migration","split":"held_out","mode":"stack","query":"which Windows UI framework should a brand new app choose instead of legacy UWP","stack":"uwp","judgments":[{"identity":{"Category":"Architecture","Guideline":"Prefer WinUI 3 for new projects"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Tests that legacy-stack guidance can recommend the successor."},
|
||||
{"id":"stack-vue-composition","split":"calibration","mode":"stack","query":"Vue Composition API for a new project","stack":"vue","judgments":[{"identity":{"Category":"Composition","Guideline":"Use Composition API for new projects"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical Vue architecture."},
|
||||
{"id":"stack-vue-pinia","split":"held_out","mode":"stack","query":"central shared Vue application state store","stack":"vue","judgments":[{"identity":{"Category":"State","Guideline":"Use Pinia for global state"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Avoids naming Pinia in the query."},
|
||||
{"id":"stack-winui-infobar","split":"calibration","mode":"stack","query":"WinUI InfoBar for status messages","stack":"winui","judgments":[{"identity":{"Category":"Controls","Guideline":"Use InfoBar for status messages"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Specific WinUI control selection."},
|
||||
{"id":"stack-winui-dispatcherqueue","split":"held_out","mode":"stack","query":"marshal a WinUI update back onto the UI thread with the modern dispatcher","stack":"winui","judgments":[{"identity":{"Category":"Threading","Guideline":"Use DispatcherQueue not Dispatcher"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Describes DispatcherQueue without copying its name."},
|
||||
{"id":"stack-wpf-property-change","split":"calibration","mode":"stack","query":"WPF INotifyPropertyChanged data binding updates","stack":"wpf","judgments":[{"identity":{"Category":"Data Binding","Guideline":"Implement INotifyPropertyChanged"},"grade":2}],"tags":["stack-positive","exact-intent"],"notes":"Canonical MVVM notification contract."},
|
||||
{"id":"stack-wpf-virtualization","split":"held_out","mode":"stack","query":"keep a huge WPF list responsive by only creating visible item containers","stack":"wpf","judgments":[{"identity":{"Category":"Performance","Guideline":"Use VirtualizingStackPanel for large lists"},"grade":2}],"tags":["stack-positive","paraphrase"],"notes":"Behavioral description of list virtualization."},
|
||||
|
||||
{"id":"auto-route-accessibility","split":"calibration","mode":"auto","query":"WCAG keyboard accessibility and visible focus","expectedRoute":"ux","judgments":[{"identity":{"Category":"Accessibility","Issue":"Keyboard Navigation","Platform":"Web"},"grade":2},{"identity":{"Category":"Interaction","Issue":"Focus States","Platform":"All"},"grade":1}],"tags":["auto-router","routing-positive"],"notes":"Specific accessibility intent should outrank generic style terms."},
|
||||
{"id":"auto-route-color","split":"calibration","mode":"auto","query":"hex color palette accent foreground destructive tokens","expectedRoute":"color","judgments":[{"identity":{"Product Type":"Design System/Component Library"},"grade":2}],"tags":["auto-router","routing-positive"],"notes":"Color vocabulary should route to palettes."},
|
||||
{"id":"auto-route-chart","split":"calibration","mode":"auto","query":"scatter chart for correlation distribution","expectedRoute":"chart","judgments":[{"identity":{"Data Type":"Correlation / Distribution"},"grade":2}],"tags":["auto-router","routing-positive"],"notes":"Explicit chart type and analytic intent."},
|
||||
{"id":"auto-route-landing","split":"held_out","mode":"auto","query":"hero testimonials CTA conversion landing page","expectedRoute":"landing","judgments":[{"identity":{"Pattern Name":"Hero + Testimonials + CTA"},"grade":2},{"identity":{"Pattern Name":"Hero + Features + CTA"},"grade":1}],"tags":["auto-router","routing-positive","held-out-route"],"notes":"Multiple landing-specific terms should dominate product routing."},
|
||||
{"id":"auto-route-fonts","split":"held_out","mode":"auto","query":"JetBrains Mono Google font family variable styles","expectedRoute":"google-fonts","judgments":[{"identity":{"Family":"JetBrains Mono"},"grade":2}],"tags":["auto-router","routing-positive","held-out-route"],"notes":"Entity lookup in the large Google Fonts catalog."},
|
||||
{"id":"auto-route-icons","split":"calibration","mode":"auto","query":"Phosphor warning icon glyph for danger status","expectedRoute":"icons","judgments":[{"identity":{"Category":"Status","Icon Name":"warning","Library":"Phosphor"},"grade":2},{"identity":{"Category":"Status","Icon Name":"warning-circle","Library":"Phosphor"},"grade":1}],"tags":["auto-router","routing-positive"],"notes":"Explicit icon/glyph vocabulary."},
|
||||
{"id":"auto-route-gsap","split":"held_out","mode":"auto","query":"GSAP ScrollTrigger stagger reveal animation","expectedRoute":"gsap","judgments":[{"identity":{"Category":"Scroll Reveal","Intensity Tier":"Standard","Trigger":"scroll (viewport enter)"},"grade":2},{"identity":{"Category":"Stagger List","Intensity Tier":"Standard","Trigger":"load or scroll"},"grade":1}],"tags":["auto-router","routing-positive","held-out-route"],"notes":"GSAP vocabulary is unambiguous even with generic animation."},
|
||||
{"id":"auto-route-react","split":"calibration","mode":"auto","query":"React Suspense waterfall bundle rerender optimization","expectedRoute":"react","judgments":[{"identity":{"Category":"Async Waterfall","Issue":"Suspense Boundaries","Platform":"React/Next.js"},"grade":2}],"tags":["auto-router","routing-positive"],"notes":"React performance terms should not route to generic UX."},
|
||||
|
||||
{"id":"negative-gibberish-alpha","split":"calibration","mode":"auto","query":"zzqqxx plmokn qvtrz","judgments":[],"tags":["hard-negative","gibberish","abstention"],"notes":"No catalog row is relevant; suggestions and returned rows are false positives for measurement."},
|
||||
{"id":"negative-gibberish-numeric","split":"held_out","mode":"auto","query":"7391 qzxv 0044 nmnq","judgments":[],"tags":["hard-negative","gibberish","abstention"],"notes":"Held-out alphanumeric noise."},
|
||||
{"id":"negative-geography-fact","split":"calibration","mode":"auto","query":"capital of Mongolia population census","judgments":[],"tags":["hard-negative","out-of-scope","abstention"],"notes":"A factual geography question, not a design request."},
|
||||
{"id":"negative-math-proof","split":"held_out","mode":"auto","query":"prove there are infinitely many prime numbers","judgments":[],"tags":["hard-negative","out-of-scope","abstention"],"notes":"A mathematics request with no relevant catalog guidance."},
|
||||
{"id":"negative-cooking","split":"calibration","mode":"auto","query":"sourdough starter feeding schedule at room temperature","judgments":[],"tags":["hard-negative","out-of-scope","abstention"],"notes":"A cooking instruction request; product-category overlap must not count as relevance."},
|
||||
{"id":"negative-biology","split":"held_out","mode":"auto","query":"photosynthesis equation for freshwater algae","judgments":[],"tags":["hard-negative","out-of-scope","abstention"],"notes":"A science question with no UI intent."},
|
||||
|
||||
{"id":"design-system-spa","split":"calibration","mode":"design-system","query":"beauty spa wellness booking landing page","judgments":[{"identity":{"Product Type":"Beauty/Spa/Wellness Service"},"grade":2}],"coherence":{"productCategory":"Beauty/Spa/Wellness Service","styleNames":["Soft UI Evolution","Neumorphism","Glassmorphism"],"patternNames":["Hero-Centric + Social Proof","Hero-Centric Design","Hero + Testimonials + CTA"],"colorMode":"light"},"tags":["design-system","coherence","readme-example"],"notes":"Industry, calming style, social proof pattern, and light palette should agree."},
|
||||
{"id":"design-system-cybersecurity-dark","split":"calibration","mode":"design-system","query":"cybersecurity threat monitoring platform dark mode","judgments":[{"identity":{"Product Type":"Cybersecurity Platform"},"grade":2}],"coherence":{"productCategory":"Cybersecurity Platform","styleNames":["Cyberpunk UI","Dark Mode (OLED)","HUD / Sci-Fi FUI"],"patternNames":["Trust & Authority + Real-Time","Trust & Authority + Conversion","Real-Time / Operations Landing","Enterprise Gateway"],"colorMode":"dark"},"tags":["design-system","coherence","dark-mode"],"notes":"Security, real-time operations, technical style, and dark palette form one coherent system."},
|
||||
{"id":"design-system-saas","split":"calibration","mode":"design-system","query":"SaaS dashboard for a B2B cloud product","judgments":[{"identity":{"Product Type":"SaaS (General)"},"grade":2},{"identity":{"Product Type":"B2B Service"},"grade":1}],"coherence":{"productCategory":"SaaS (General)","styleNames":["Glassmorphism","Flat Design","Minimalism & Swiss Style","Soft UI Evolution"],"patternNames":["Hero + Features + CTA","Feature-Rich Showcase"],"colorMode":"light"},"tags":["design-system","coherence","readme-example"],"notes":"Locks the README's broad SaaS example to a maintainable B2B system."},
|
||||
{"id":"design-system-healthcare","split":"calibration","mode":"design-system","query":"accessible healthcare analytics dashboard for patients","judgments":[{"identity":{"Product Type":"Healthcare App"},"grade":2},{"identity":{"Product Type":"Patient Portal / Health Records"},"grade":1}],"coherence":{"productCategory":"Healthcare App","styleNames":["Accessible & Ethical","Inclusive Design","Neumorphism","Soft UI Evolution"],"patternNames":["Social Proof-Focused","Hero + Testimonials + CTA","Trust & Authority + Conversion"],"colorMode":"light"},"tags":["design-system","coherence","accessibility","readme-example"],"notes":"Accessibility and patient trust are stronger constraints than decorative dashboard styling."},
|
||||
{"id":"design-system-portfolio-dark","split":"held_out","mode":"design-system","query":"creative portfolio website with dark mode and scroll storytelling","judgments":[{"identity":{"Product Type":"Portfolio/Personal"},"grade":2}],"coherence":{"productCategory":"Portfolio/Personal","styleNames":["Motion-Driven","Brutalism","Dark Mode (OLED)","Minimalism & Swiss Style","Interactive Cursor Design"],"patternNames":["Storytelling-Driven","Scroll-Triggered Storytelling","Portfolio Grid","Horizontal Scroll Journey"],"colorMode":"dark","colorProductTypes":["Portfolio/Personal"]},"tags":["design-system","coherence","dark-mode","readme-example"],"notes":"Explicit creative intent activates the curated Brutalism rule; the derived dark surface must retain the Portfolio/Personal palette identity."},
|
||||
{"id":"design-system-fintech-dark","split":"held_out","mode":"design-system","query":"fintech banking app with a secure dark dashboard","judgments":[{"identity":{"Product Type":"Fintech/Crypto"},"grade":2},{"identity":{"Product Type":"Banking/Traditional Finance"},"grade":1}],"coherence":{"productCategory":"Fintech/Crypto","styleNames":["Dark Mode (OLED)","Glassmorphism","Accessible & Ethical","Minimalism & Swiss Style"],"patternNames":["Trust & Authority","Trust & Authority + Conversion","Enterprise Gateway"],"colorMode":"dark"},"tags":["design-system","coherence","dark-mode","readme-example"],"notes":"Explicit dark mode must not conflict with the palette or anti-pattern advice."},
|
||||
{"id":"design-system-spa-paraphrase","split":"held_out","mode":"design-system","query":"calming salon for massages facials and appointment reservations","judgments":[{"identity":{"Product Type":"Beauty/Spa/Wellness Service"},"grade":2},{"identity":{"Product Type":"Booking & Appointment App"},"grade":1}],"coherence":{"productCategory":"Beauty/Spa/Wellness Service","styleNames":["Soft UI Evolution","Neumorphism","Organic Biophilic","Nature Distilled"],"patternNames":["Hero-Centric + Social Proof","Hero-Centric Design","Hero + Testimonials + CTA"],"colorMode":"light"},"tags":["design-system","coherence","paraphrase"],"notes":"Held-out industry paraphrase without the exact words beauty, spa, or wellness."},
|
||||
{"id":"design-system-cyber-typo","split":"held_out","mode":"design-system","query":"cybersecurty operatons center with live threat alerts and OLED UI","judgments":[{"identity":{"Product Type":"Cybersecurity Platform"},"grade":2}],"coherence":{"productCategory":"Cybersecurity Platform","styleNames":["Cyberpunk UI","Dark Mode (OLED)","HUD / Sci-Fi FUI"],"patternNames":["Trust & Authority + Real-Time","Real-Time / Operations Landing","Trust & Authority + Conversion"],"colorMode":"dark"},"tags":["design-system","coherence","typo","dark-mode"],"notes":"Typos plus an OLED constraint test end-to-end recovery and agreement."}
|
||||
]
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"status": "provisional-baseline-regression-gate",
|
||||
"baselineRevision": "97eb2a2",
|
||||
"runtimeFingerprint": "0d096d52499c2dd84ef80b2e45a1793bc4ed7c9be8fe5948b08151790959b376",
|
||||
"oracleFingerprint": "02e7226428c64a995ec24ae104ce4d186f5f91bf0f062556e90ef9b0fcb6d67a",
|
||||
"approvingMaintainer": "repository maintainer approved Phase 1 testing checkpoint; second judgment review remains required before final-target promotion",
|
||||
"units": "All metrics are ratios in [0,1]. Precision treats missing ranks as non-relevant. Negative abstention is measured across the hard-negative by domain/stack cross-product.",
|
||||
"splitPolicy": {
|
||||
"calibration": "May be inspected while tuning retrieval.",
|
||||
"held_out": "Must not be inspected to tune ranking weights; used for final confirmation.",
|
||||
"varianceAndConfidence": "Corpus is deterministic and has no sampling variance. Report exact ratios and sample counts; do not claim population confidence intervals from this curated set."
|
||||
},
|
||||
"tolerancePolicy": "A 1e-12 numeric tolerance permits floating-point representation only. Current known failures remain visible in relevance-baseline.json; the gate prevents regressions and does not claim final quality targets are met.",
|
||||
"metrics": {
|
||||
"routingAccuracy": {"floor": 0.875, "tolerance": 1e-12},
|
||||
"precisionAt1": {"floor": 0.7631578947368421, "tolerance": 1e-12},
|
||||
"precisionAt3": {"floor": 0.3815789473684211, "tolerance": 1e-12},
|
||||
"mrrAt3": {"floor": 0.8355263157894737, "tolerance": 1e-12},
|
||||
"ndcgAt3": {"floor": 0.8366350964266653, "tolerance": 1e-12},
|
||||
"negativeAbstention": {"floor": 0.9117647058823529, "tolerance": 1e-12},
|
||||
"typoRecoveryAt3": {"floor": 1.0, "tolerance": 1e-12},
|
||||
"designSystemCoherence": {"floor": 0.71875, "tolerance": 1e-12}
|
||||
},
|
||||
"sampleMinimums": {
|
||||
"cases": 90,
|
||||
"retrieval": 76,
|
||||
"routing": 8,
|
||||
"negativeChecks": 204,
|
||||
"typo": 5,
|
||||
"designSystem": 8,
|
||||
"domain:style": 2,
|
||||
"domain:color": 2,
|
||||
"domain:chart": 2,
|
||||
"domain:landing": 2,
|
||||
"domain:product": 2,
|
||||
"domain:ux": 2,
|
||||
"domain:typography": 2,
|
||||
"domain:icons": 2,
|
||||
"domain:gsap": 2,
|
||||
"domain:react": 2,
|
||||
"domain:web": 2,
|
||||
"domain:google-fonts": 2
|
||||
},
|
||||
"splits": {
|
||||
"calibration": {
|
||||
"metrics": {
|
||||
"routingAccuracy": {"floor": 0.8, "tolerance": 1e-12},
|
||||
"precisionAt1": {"floor": 0.8974358974358975, "tolerance": 1e-12},
|
||||
"precisionAt3": {"floor": 0.40170940170940167, "tolerance": 1e-12},
|
||||
"mrrAt3": {"floor": 0.9230769230769231, "tolerance": 1e-12},
|
||||
"ndcgAt3": {"floor": 0.9103072109232644, "tolerance": 1e-12},
|
||||
"negativeAbstention": {"floor": 0.9411764705882353, "tolerance": 1e-12},
|
||||
"typoRecoveryAt3": {"floor": 0.0, "tolerance": 1e-12},
|
||||
"designSystemCoherence": {"floor": 0.6875, "tolerance": 1e-12}
|
||||
},
|
||||
"sampleMinimums": {"cases": 46, "retrieval": 39, "routing": 5, "negativeChecks": 102, "typo": 0, "designSystem": 4}
|
||||
},
|
||||
"held_out": {
|
||||
"metrics": {
|
||||
"routingAccuracy": {"floor": 1.0, "tolerance": 1e-12},
|
||||
"precisionAt1": {"floor": 0.6216216216216216, "tolerance": 1e-12},
|
||||
"precisionAt3": {"floor": 0.36036036036036034, "tolerance": 1e-12},
|
||||
"mrrAt3": {"floor": 0.7432432432432432, "tolerance": 1e-12},
|
||||
"ndcgAt3": {"floor": 0.7589807054707906, "tolerance": 1e-12},
|
||||
"negativeAbstention": {"floor": 0.8823529411764706, "tolerance": 1e-12},
|
||||
"typoRecoveryAt3": {"floor": 1.0, "tolerance": 1e-12},
|
||||
"designSystemCoherence": {"floor": 0.75, "tolerance": 1e-12}
|
||||
},
|
||||
"sampleMinimums": {"cases": 44, "retrieval": 37, "routing": 3, "negativeChecks": 102, "typo": 5, "designSystem": 4}
|
||||
}
|
||||
},
|
||||
"lockedCases": {
|
||||
"domain-style-glassmorphism": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-color-spa": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-chart-time-series": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-landing-pricing": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-product-spa": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-ux-keyboard-focus": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-typography-luxury": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-icons-search": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-gsap-scroll-reveal": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-react-parallel-promises": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-web-icon-label": {"withinTop": 1, "minimumGrade": 2},
|
||||
"domain-google-fonts-inter": {"withinTop": 1, "minimumGrade": 2},
|
||||
"stack-swiftui-navigation": {"withinTop": 1, "minimumGrade": 2},
|
||||
"stack-threejs-orbitcontrols": {"withinTop": 1, "minimumGrade": 2},
|
||||
"stack-uwp-migration": {"withinTop": 1, "minimumGrade": 2},
|
||||
"stack-winui-dispatcherqueue": {"withinTop": 1, "minimumGrade": 2}
|
||||
},
|
||||
"proposedFinalTargets": {
|
||||
"routingAccuracy": 0.93,
|
||||
"precisionAt1": 0.8,
|
||||
"mrrAt3": 0.88,
|
||||
"ndcgAt3": 0.92,
|
||||
"negativeAbstention": 0.95,
|
||||
"typoRecoveryAt3": 0.85,
|
||||
"designSystemCoherence": 0.9
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline contract tests for deterministic upstream catalog refreshes."""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO = next(
|
||||
parent for parent in Path(__file__).resolve().parents
|
||||
if all((parent / "scripts" / script).is_file() for script in (
|
||||
"refresh-google-fonts.py", "refresh-icon-catalog.py",
|
||||
))
|
||||
)
|
||||
FIXTURES = Path(__file__).parent / "fixtures" / "catalogs"
|
||||
FONT_SCRIPT = REPO / "scripts" / "refresh-google-fonts.py"
|
||||
ICON_SCRIPT = REPO / "scripts" / "refresh-icon-catalog.py"
|
||||
|
||||
|
||||
class CatalogRefreshTest(unittest.TestCase):
|
||||
def run_command(self, *args, env=None):
|
||||
return subprocess.run(
|
||||
[sys.executable, *map(str, args)],
|
||||
cwd=REPO,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def font_args(self, directory, api=None, metadata=None, approve=True, existing=None, overrides=None):
|
||||
args = [
|
||||
FONT_SCRIPT,
|
||||
"--api-input", api or FIXTURES / "google-api.json",
|
||||
"--metadata-input", metadata or FIXTURES / "google-metadata.json",
|
||||
"--existing-csv", existing or FIXTURES / "google-existing.csv",
|
||||
"--overrides", overrides or FIXTURES / "google-overrides.json",
|
||||
"--output-csv", directory / "google-fonts.csv",
|
||||
"--license-output", directory / "google-font-licenses.json",
|
||||
"--verified-at", "2026-08-13",
|
||||
"--metadata-revision", "fixture-catalogs-v1",
|
||||
"--expected-count", "2",
|
||||
]
|
||||
if approve:
|
||||
args.append("--approve-changes")
|
||||
return args
|
||||
|
||||
def icon_args(self, directory, source=None, curated=None, package=None, react_exports=None):
|
||||
return [
|
||||
ICON_SCRIPT,
|
||||
"--input", source or FIXTURES / "phosphor-core.json",
|
||||
"--package-json", package or FIXTURES / "phosphor-package.json",
|
||||
"--react-package-json", FIXTURES / "phosphor-react-package.json",
|
||||
"--react-exports-input", react_exports or FIXTURES / "phosphor-react-exports.json",
|
||||
"--curated-csv", curated or FIXTURES / "icons-curated.csv",
|
||||
"--output", directory / "phosphor-icons-upstream.json",
|
||||
"--verified-at", "2026-08-13",
|
||||
"--expected-count", "2",
|
||||
]
|
||||
|
||||
def test_live_font_refresh_requires_environment_key(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
args = self.font_args(directory)
|
||||
args[1:3] = ["--live"]
|
||||
env = dict(os.environ)
|
||||
env.pop("GOOGLE_FONTS_API_KEY", None)
|
||||
result = self.run_command(*args, env=env)
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn("GOOGLE_FONTS_API_KEY is required for --live", result.stderr)
|
||||
self.assertIn("use --api-input for offline CI", result.stderr)
|
||||
|
||||
def test_font_refresh_is_deterministic_and_preserves_reviewed_fields(self):
|
||||
with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
|
||||
first_path, second_path = Path(first), Path(second)
|
||||
self.assertEqual(0, self.run_command(*self.font_args(first_path)).returncode)
|
||||
self.assertEqual(0, self.run_command(*self.font_args(second_path)).returncode)
|
||||
self.assertEqual(
|
||||
(first_path / "google-fonts.csv").read_bytes(),
|
||||
(second_path / "google-fonts.csv").read_bytes(),
|
||||
)
|
||||
self.assertEqual(
|
||||
(first_path / "google-font-licenses.json").read_bytes(),
|
||||
(second_path / "google-font-licenses.json").read_bytes(),
|
||||
)
|
||||
with (first_path / "google-fonts.csv").open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
csv_bytes = (first_path / "google-fonts.csv").read_bytes()
|
||||
license_bytes = (first_path / "google-font-licenses.json").read_bytes()
|
||||
licenses = json.loads(license_bytes)
|
||||
self.assertEqual(["Alpha Sans", "Zeta Serif"], [row["Family"] for row in rows])
|
||||
self.assertEqual("Sans Serif", rows[0]["Stroke"])
|
||||
self.assertEqual("approved override keywords", rows[0]["Keywords"])
|
||||
self.assertEqual("400 | 400i | 500", rows[0]["Styles"])
|
||||
self.assertEqual("wght: 100..900", rows[0]["Variable Axes"])
|
||||
self.assertEqual(["OFL", "APACHE2"], [item["license"] for item in licenses["families"]])
|
||||
self.assertEqual(["Alpha Sans", "Zeta Serif"], [item["name"] for item in licenses["families"]])
|
||||
self.assertEqual("fixture-catalogs-v1", licenses["source"]["revision"])
|
||||
self.assertTrue(all(item["status"] == "active" for item in licenses["families"]))
|
||||
self.assertTrue(all(item["verifiedAt"] == "2026-08-13" for item in licenses["families"]))
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
reused = Path(raw)
|
||||
metadata_path = reused / "metadata.json"
|
||||
metadata_path.write_bytes(license_bytes)
|
||||
result = self.run_command(*self.font_args(
|
||||
reused, metadata=metadata_path
|
||||
))
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual(
|
||||
csv_bytes, (reused / "google-fonts.csv").read_bytes(),
|
||||
)
|
||||
|
||||
def test_font_refresh_rejects_schema_size_dates_and_licenses(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
api = json.loads((FIXTURES / "google-api.json").read_text())
|
||||
metadata = json.loads((FIXTURES / "google-metadata.json").read_text())
|
||||
cases = []
|
||||
wrong_schema = dict(api)
|
||||
wrong_schema["kind"] = "unexpected"
|
||||
cases.append((wrong_schema, metadata, "webfonts#webfontList"))
|
||||
cases.append(({**api, "items": api["items"][:1]}, metadata, "expected 2 items"))
|
||||
bad_url = json.loads(json.dumps(api))
|
||||
bad_url["items"][0]["files"]["regular"] = "https://example.com/font.ttf"
|
||||
cases.append((bad_url, metadata, "https://fonts.gstatic.com"))
|
||||
bad_date = json.loads(json.dumps(api))
|
||||
bad_date["items"][0]["lastModified"] = "1970-01-01"
|
||||
cases.append((bad_date, metadata, "suspicious date"))
|
||||
bad_license = json.loads(json.dumps(metadata))
|
||||
bad_license["families"][0]["license"] = "UNKNOWN"
|
||||
cases.append((api, bad_license, "invalid or missing official license"))
|
||||
for index, (api_value, metadata_value, error) in enumerate(cases):
|
||||
api_path, metadata_path = directory / f"api-{index}.json", directory / f"metadata-{index}.json"
|
||||
api_path.write_text(json.dumps(api_value))
|
||||
metadata_path.write_text(json.dumps(metadata_value))
|
||||
result = self.run_command(*self.font_args(directory, api_path, metadata_path))
|
||||
with self.subTest(error=error):
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn(error, result.stderr)
|
||||
|
||||
def test_font_refresh_fails_closed_on_concurrency_or_interrupted_pair(self):
|
||||
for sentinel, error in (
|
||||
(".google-font-refresh.lock", "another refresh is already running"),
|
||||
(".google-font-refresh.incomplete.json", "incomplete prior refresh"),
|
||||
):
|
||||
with self.subTest(sentinel=sentinel), tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
(directory / sentinel).write_text("occupied\n", encoding="utf-8")
|
||||
result = self.run_command(*self.font_args(directory))
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn(error, result.stderr)
|
||||
self.assertFalse((directory / "google-fonts.csv").exists())
|
||||
self.assertFalse((directory / "google-font-licenses.json").exists())
|
||||
|
||||
def test_catalog_cross_check_uses_explicit_schema_without_font_file_urls(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
args = self.font_args(directory)
|
||||
args[1:3] = ["--catalog-input", FIXTURES / "google-catalog.json"]
|
||||
result = self.run_command(*args)
|
||||
with (directory / "google-fonts.csv").open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual(["Alpha Sans", "Zeta Serif"], [row["Family"] for row in rows])
|
||||
self.assertEqual("Geometric", rows[0]["Classifications"])
|
||||
self.assertEqual("42", rows[0]["Popularity Rank"])
|
||||
self.assertEqual("latin | vietnamese", rows[0]["Subsets"])
|
||||
|
||||
def test_official_metadata_checkout_is_strict_and_reusable(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
metadata_root = directory / "google-fonts"
|
||||
family_dir = metadata_root / "ofl" / "alphasans"
|
||||
family_dir.mkdir(parents=True)
|
||||
(family_dir / "METADATA.pb").write_text(
|
||||
'name: "Alpha Sans"\n'
|
||||
'designer: "Alpha Designer"\n'
|
||||
'license: "OFL"\n'
|
||||
'date_added: "2024-01-02"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
args = self.font_args(directory)
|
||||
metadata_index = args.index("--metadata-input")
|
||||
args[metadata_index:metadata_index + 2] = ["--metadata-root", metadata_root]
|
||||
result = self.run_command(*args)
|
||||
self.assertEqual(2, result.returncode)
|
||||
self.assertIn("fewer than 90%", result.stderr)
|
||||
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
metadata_root = directory / "google-fonts"
|
||||
for slug, family, license_name in (
|
||||
("alphasans", "Alpha Sans", "OFL"),
|
||||
("zetaserif", "Zeta Serif", "APACHE2"),
|
||||
):
|
||||
family_dir = metadata_root / "ofl" / slug
|
||||
family_dir.mkdir(parents=True)
|
||||
(family_dir / "METADATA.pb").write_text(
|
||||
f'name: "{family}"\n'
|
||||
f'designer: "{family} Designer"\n'
|
||||
f'license: "{license_name}"\n'
|
||||
'date_added: "2024-01-02"\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
args = self.font_args(directory)
|
||||
metadata_index = args.index("--metadata-input")
|
||||
args[metadata_index:metadata_index + 2] = ["--metadata-root", metadata_root]
|
||||
result = self.run_command(*args)
|
||||
licenses = json.loads((directory / "google-font-licenses.json").read_text())
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual(["Alpha Sans", "Zeta Serif"], [item["name"] for item in licenses["families"]])
|
||||
|
||||
def test_catalog_rejects_bool_rank_duplicate_axis_and_unreviewed_addition(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
catalog = json.loads((FIXTURES / "google-catalog.json").read_text())
|
||||
invalid_rank = json.loads(json.dumps(catalog))
|
||||
invalid_rank["familyMetadataList"][0]["popularity"] = True
|
||||
rank_path = directory / "rank.json"
|
||||
rank_path.write_text(json.dumps(invalid_rank))
|
||||
rank_args = self.font_args(directory)
|
||||
rank_args[1:3] = ["--catalog-input", rank_path]
|
||||
rank_result = self.run_command(*rank_args)
|
||||
invalid_axis = json.loads(json.dumps(catalog))
|
||||
axis = invalid_axis["familyMetadataList"][1]["axes"][0]
|
||||
invalid_axis["familyMetadataList"][1]["axes"].append(dict(axis))
|
||||
axis_path = directory / "axis.json"
|
||||
axis_path.write_text(json.dumps(invalid_axis))
|
||||
axis_args = self.font_args(directory)
|
||||
axis_args[1:3] = ["--catalog-input", axis_path]
|
||||
axis_result = self.run_command(*axis_args)
|
||||
existing = directory / "existing.csv"
|
||||
lines = (FIXTURES / "google-existing.csv").read_text().splitlines()
|
||||
existing.write_text("\n".join(lines[:2]) + "\n")
|
||||
approval_result = self.run_command(*self.font_args(directory, approve=False, existing=existing))
|
||||
bad_overrides = directory / "overrides.json"
|
||||
bad_overrides.write_text('{"families":{"Unknown Font":{"Keywords":"bad"}}}')
|
||||
override_result = self.run_command(*self.font_args(directory, overrides=bad_overrides))
|
||||
self.assertIn("invalid popularity", rank_result.stderr)
|
||||
self.assertIn("duplicate axis tags", axis_result.stderr)
|
||||
self.assertIn("family-set changes require --approve-changes", approval_result.stderr)
|
||||
self.assertIn("Zeta Serif", approval_result.stdout)
|
||||
self.assertFalse((directory / "google-fonts.csv").exists())
|
||||
self.assertIn("overrides target unknown families", override_result.stderr)
|
||||
|
||||
def test_explicit_license_exclusion_is_reported_and_not_promoted(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
metadata = json.loads((FIXTURES / "google-metadata.json").read_text())
|
||||
metadata["families"] = metadata["families"][:1]
|
||||
metadata["excludedFamilies"] = [{
|
||||
"name": "Alpha Sans", "reason": "No matching official METADATA.pb",
|
||||
"source": "https://github.com/google/fonts",
|
||||
}]
|
||||
metadata_path = directory / "metadata.json"
|
||||
metadata_path.write_text(json.dumps(metadata))
|
||||
result = self.run_command(*self.font_args(directory, metadata=metadata_path))
|
||||
report = json.loads(result.stdout)
|
||||
licenses = json.loads((directory / "google-font-licenses.json").read_text())
|
||||
with (directory / "google-fonts.csv").open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
self.assertEqual(["Zeta Serif"], [row["Family"] for row in rows])
|
||||
self.assertEqual("needs-review", report["excludedFamilies"][0]["status"])
|
||||
self.assertEqual("needs-review", licenses["excludedFamilies"][0]["status"])
|
||||
|
||||
def test_exclusion_sources_match_offline_validator_policy(self):
|
||||
metadata = json.loads((FIXTURES / "google-metadata.json").read_text())
|
||||
allowed = (
|
||||
"https://fonts.google.com/specimen/Alpha+Sans",
|
||||
"https://github.com/google/fonts",
|
||||
"https://github.com/google/fonts/tree/main/ofl/alphasans",
|
||||
)
|
||||
rejected = (
|
||||
"https://example.com/google/fonts",
|
||||
"https://github.com/other/fonts",
|
||||
"https://fonts.google.com:444/specimen/Alpha+Sans",
|
||||
)
|
||||
for index, source in enumerate((*allowed, *rejected)):
|
||||
with self.subTest(source=source), tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
candidate = json.loads(json.dumps(metadata))
|
||||
candidate["families"] = candidate["families"][1:]
|
||||
candidate["excludedFamilies"] = [{
|
||||
"name": "Zeta Serif",
|
||||
"reason": "No matching official METADATA.pb",
|
||||
"source": source,
|
||||
}]
|
||||
metadata_path = directory / f"metadata-{index}.json"
|
||||
metadata_path.write_text(json.dumps(candidate))
|
||||
result = self.run_command(*self.font_args(directory, metadata=metadata_path))
|
||||
self.assertEqual(source in allowed, result.returncode == 0, result.stderr)
|
||||
|
||||
def test_icon_manifest_normalizes_and_records_all_import_forms(self):
|
||||
with tempfile.TemporaryDirectory() as first, tempfile.TemporaryDirectory() as second:
|
||||
first_path, second_path = Path(first), Path(second)
|
||||
self.assertEqual(0, self.run_command(*self.icon_args(first_path)).returncode)
|
||||
self.assertEqual(0, self.run_command(*self.icon_args(second_path)).returncode)
|
||||
output = first_path / "phosphor-icons-upstream.json"
|
||||
self.assertEqual(output.read_bytes(), (second_path / output.name).read_bytes())
|
||||
manifest = json.loads(output.read_text())
|
||||
self.assertEqual("2.1.1", manifest["source"]["version"])
|
||||
self.assertEqual(("active", "2026-08-13"), (manifest["status"], manifest["verifiedAt"]))
|
||||
self.assertEqual(["thin", "light", "regular", "bold", "fill", "duotone"], manifest["weights"])
|
||||
self.assertEqual(["acorn", "arrow-left"], [icon["name"] for icon in manifest["icons"]])
|
||||
arrow = manifest["icons"][1]
|
||||
self.assertEqual(["arrows", "navigation"], arrow["categories"])
|
||||
self.assertIn('from "@phosphor-icons/react"', arrow["clientImport"])
|
||||
self.assertIn('from "@phosphor-icons/react/ssr"', arrow["ssrImport"])
|
||||
self.assertEqual(2, manifest["curatedValidatedCount"])
|
||||
|
||||
def test_icon_refresh_rejects_invalid_schema_size_and_curated_import(self):
|
||||
with tempfile.TemporaryDirectory() as raw:
|
||||
directory = Path(raw)
|
||||
icons = json.loads((FIXTURES / "phosphor-core.json").read_text())
|
||||
invalid = json.loads(json.dumps(icons))
|
||||
del invalid[0]["pascal_name"]
|
||||
source = directory / "invalid.json"
|
||||
source.write_text(json.dumps(invalid))
|
||||
schema_result = self.run_command(*self.icon_args(directory, source))
|
||||
size_args = self.icon_args(directory)
|
||||
size_args[-1] = "3"
|
||||
size_result = self.run_command(*size_args)
|
||||
curated = (FIXTURES / "icons-curated.csv").read_text().replace("{ Acorn }", "{ Horse }")
|
||||
curated_path = directory / "icons.csv"
|
||||
curated_path.write_text(curated)
|
||||
import_result = self.run_command(*self.icon_args(directory, curated=curated_path))
|
||||
package_path = directory / "package.json"
|
||||
package_path.write_text('{"name":"@phosphor-icons/core","version":"2.2.0"}')
|
||||
version_result = self.run_command(*self.icon_args(directory, package=package_path))
|
||||
alias_collision = json.loads(json.dumps(icons))
|
||||
alias_collision[1]["alias"] = {"name": "acorn", "pascal_name": "BackArrow"}
|
||||
alias_path = directory / "alias.json"
|
||||
alias_path.write_text(json.dumps(alias_collision))
|
||||
alias_result = self.run_command(*self.icon_args(directory, source=alias_path))
|
||||
exports = json.loads((FIXTURES / "phosphor-react-exports.json").read_text())
|
||||
exports["ssr"].remove("Acorn")
|
||||
exports_path = directory / "exports.json"
|
||||
exports_path.write_text(json.dumps(exports))
|
||||
exports_result = self.run_command(*self.icon_args(directory, react_exports=exports_path))
|
||||
self.assertIn("invalid official IconEntry schema", schema_result.stderr)
|
||||
self.assertIn("expected 3 icons", size_result.stderr)
|
||||
self.assertIn("import component does not match", import_result.stderr)
|
||||
self.assertIn("version must be 2.1.1", version_result.stderr)
|
||||
self.assertIn("alias collides", alias_result.stderr)
|
||||
self.assertIn("React exports missing", exports_result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+216
-7
@@ -11,16 +11,21 @@ or directly:
|
||||
python scripts/tests/test_core.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import core
|
||||
from core import BM25, detect_domain, search, search_stack, CSV_CONFIG, AVAILABLE_STACKS
|
||||
from design_system import generate_design_system, persist_design_system, DesignSystemGenerator
|
||||
from design_system import DesignSystemGenerator, generate_design_system
|
||||
|
||||
|
||||
class TestTokenizer(unittest.TestCase):
|
||||
@@ -42,11 +47,80 @@ class TestTokenizer(unittest.TestCase):
|
||||
self.assertEqual(bm25.tokenize("e-commerce store"), bm25.tokenize("ecommerce store"))
|
||||
self.assertEqual(bm25.tokenize("dark-mode toggle"), bm25.tokenize("dark toggle"))
|
||||
|
||||
def test_boundary_safe_nav_normalization_preserves_existing_words(self):
|
||||
bm25 = BM25()
|
||||
tokens = bm25.tokenize("nav navigation navbar")
|
||||
self.assertIn("navigation", tokens)
|
||||
self.assertIn("navbar", tokens)
|
||||
self.assertNotIn("navigationigation", tokens)
|
||||
self.assertNotIn("navigationbar", tokens)
|
||||
|
||||
def test_punctuation_and_uk_variants_normalize_to_canonical_tokens(self):
|
||||
bm25 = BM25()
|
||||
tokens = bm25.tokenize("colour, organisation; behaviour customisation")
|
||||
for expected in ("color", "organization", "behavior", "customization"):
|
||||
self.assertIn(expected, tokens)
|
||||
|
||||
|
||||
class TestBm25CoreBehavior(unittest.TestCase):
|
||||
def test_empty_documents_produce_no_scores_or_vocab(self):
|
||||
bm25 = BM25()
|
||||
bm25.fit([])
|
||||
self.assertEqual(bm25.score("anything"), [])
|
||||
self.assertEqual(bm25.vocabulary(), [])
|
||||
|
||||
def test_bm25_cache_rebuilds_after_file_mtime_changes(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "search.csv"
|
||||
path.write_text("Name,Keywords\nAlpha,alpha token\n", encoding="utf-8")
|
||||
results_a, bm25_a = core._search_csv(path, ["Name", "Keywords"], ["Name"], "alpha", 1)
|
||||
self.assertEqual(results_a[0]["Name"], "Alpha")
|
||||
|
||||
path.write_text("Name,Keywords\nBeta,beta token\n", encoding="utf-8")
|
||||
stat = path.stat()
|
||||
os.utime(path, ns=(stat.st_atime_ns + 1_000_000_000, stat.st_mtime_ns + 1_000_000_000))
|
||||
|
||||
results_b, bm25_b = core._search_csv(path, ["Name", "Keywords"], ["Name"], "beta", 1)
|
||||
self.assertEqual(results_b[0]["Name"], "Beta")
|
||||
self.assertIsNot(bm25_a, bm25_b)
|
||||
|
||||
def test_search_uses_one_verified_rows_and_index_snapshot(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "search.csv"
|
||||
path.write_text("Name,Keywords\nAlpha,alpha token\n", encoding="utf-8")
|
||||
original_get_bm25 = core._get_bm25
|
||||
replaced = False
|
||||
|
||||
def replace_after_read(filepath, search_cols, data, signature=None,
|
||||
cache_variant=""):
|
||||
nonlocal replaced
|
||||
if not replaced:
|
||||
path.write_text("Name,Keywords\nBeta,beta token\n", encoding="utf-8")
|
||||
replaced = True
|
||||
return original_get_bm25(
|
||||
filepath, search_cols, data, signature, cache_variant)
|
||||
|
||||
with patch.object(core, "_get_bm25", side_effect=replace_after_read):
|
||||
results, _, _ = core._search_csv_detailed(
|
||||
path, ["Name", "Keywords"], ["Name"], "alpha", 1)
|
||||
self.assertEqual(results[0]["Name"], "Alpha")
|
||||
|
||||
results, _, _ = core._search_csv_detailed(
|
||||
path, ["Name", "Keywords"], ["Name"], "beta", 1)
|
||||
self.assertEqual(results[0]["Name"], "Beta")
|
||||
|
||||
|
||||
class TestSearchDomains(unittest.TestCase):
|
||||
"""Known query -> expected top-domain sanity checks (not exact-row pinning,
|
||||
since data can grow; these assert the engine still finds *something*
|
||||
relevant for each domain's core vocabulary)."""
|
||||
def test_read_failure_is_not_reported_as_a_search_result(self):
|
||||
failure = UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid")
|
||||
with patch("core._load_csv_snapshot", side_effect=failure):
|
||||
domain = search("palette", domain="color", max_results=1)
|
||||
stack = search_stack("component", "react", max_results=1)
|
||||
for result in (domain, stack):
|
||||
self.assertEqual(0, result["count"])
|
||||
self.assertEqual([], result["results"])
|
||||
self.assertRegex(result["error"], r"^Unable to read search data:")
|
||||
self.assertNotIn("invalid", result["error"])
|
||||
|
||||
def test_ui_is_searchable_in_style_domain(self):
|
||||
result = search("ui minimalism", domain="style", max_results=1)
|
||||
@@ -62,12 +136,62 @@ class TestSearchDomains(unittest.TestCase):
|
||||
self.assertIn("suggestions", result)
|
||||
self.assertNotIn("error", result)
|
||||
|
||||
def test_hard_negative_query_abstains_across_registered_domains_and_stacks(self):
|
||||
query = "sourdough starter crumb fermentation"
|
||||
for domain in CSV_CONFIG:
|
||||
with self.subTest(kind="domain", name=domain):
|
||||
self.assertEqual(search(query, domain=domain, max_results=1)["count"], 0)
|
||||
for stack in AVAILABLE_STACKS:
|
||||
with self.subTest(kind="stack", name=stack):
|
||||
self.assertEqual(search_stack(query, stack, max_results=1)["count"], 0)
|
||||
|
||||
def test_typo_suggestions_are_deterministic_and_retryable(self):
|
||||
first = search("testimonal", domain="landing", max_results=3)
|
||||
second = search("testimonal", domain="landing", max_results=3)
|
||||
self.assertEqual(first["count"], 0)
|
||||
self.assertEqual(first.get("suggestions"), second.get("suggestions"))
|
||||
self.assertTrue(first["suggestions"], "typo path should return at least one deterministic suggestion")
|
||||
|
||||
retry = search(first["suggestions"][0], domain="landing", max_results=3)
|
||||
self.assertGreater(retry["count"], 0)
|
||||
|
||||
def test_suggestions_never_repeat_the_input_or_offer_a_dead_first_retry(self):
|
||||
pricing = search("pricing", domain="landing", max_results=3)
|
||||
self.assertNotIn("pricing", pricing.get("suggestions", []))
|
||||
|
||||
minimal = search("minimal", domain="style", max_results=3)
|
||||
self.assertEqual(1, minimal["count"])
|
||||
self.assertEqual(
|
||||
"minimalism-and-swiss-style", minimal["results"][0]["Style ID"]
|
||||
)
|
||||
|
||||
def test_unknown_programmatic_domain_keeps_legacy_style_fallback(self):
|
||||
result = search("minimalism", domain="unknown", max_results=1)
|
||||
self.assertEqual(result["domain"], "unknown")
|
||||
self.assertEqual(result["file"], CSV_CONFIG["style"]["file"])
|
||||
self.assertGreater(result["count"], 0)
|
||||
|
||||
def test_unsupported_icon_library_abstains_instead_of_returning_other_library(self):
|
||||
result = search("lucide icon", diagnostics=True)
|
||||
self.assertEqual(result["domain"], "icons")
|
||||
self.assertEqual(result["count"], 0)
|
||||
self.assertEqual(result["diagnostics"]["reason"], "unsupported-library")
|
||||
|
||||
def test_every_configured_domain_file_exists_and_is_searchable(self):
|
||||
for domain, config in CSV_CONFIG.items():
|
||||
with self.subTest(domain=domain):
|
||||
result = search("design", domain=domain, max_results=1)
|
||||
self.assertNotIn("error", result, f"domain '{domain}' failed: {result.get('error')}")
|
||||
|
||||
def test_chart_output_keeps_legacy_grade_during_risk_migration(self):
|
||||
result = search("time series chart", domain="chart", max_results=1)
|
||||
self.assertEqual(result["count"], 1)
|
||||
self.assertEqual(
|
||||
result["results"][0]["Accessibility Grade"],
|
||||
"deprecated: use Accessibility Risk",
|
||||
)
|
||||
self.assertIn("Accessibility Risk", result["results"][0])
|
||||
|
||||
def test_every_stack_file_exists_and_is_searchable(self):
|
||||
for stack in AVAILABLE_STACKS:
|
||||
with self.subTest(stack=stack):
|
||||
@@ -83,16 +207,67 @@ class TestDomainDetection(unittest.TestCase):
|
||||
self.assertEqual(detect_domain("accessibility contrast wcag"), "ux")
|
||||
|
||||
def test_ambiguous_query_returns_runner_up(self):
|
||||
domain, runner_up = detect_domain("font pairing elegant crypto", return_scores=True)
|
||||
domain, _ = detect_domain("font pairing elegant crypto", return_scores=True)
|
||||
self.assertIsNotNone(domain)
|
||||
# runner_up may be None if the winning domain has no close second --
|
||||
# this just verifies the call shape works without raising.
|
||||
|
||||
def test_empty_query_falls_back_to_style(self):
|
||||
self.assertEqual(detect_domain("...!!!???"), "style")
|
||||
|
||||
def test_router_prioritizes_color_intent_over_generic_product_terms(self):
|
||||
self.assertEqual(detect_domain("semantic color tokens palette"), "color")
|
||||
|
||||
def test_router_prioritizes_icons_when_icon_library_and_icon_intent_present(self):
|
||||
self.assertEqual(detect_domain("lucide search icon outline"), "icons")
|
||||
|
||||
def test_router_prioritizes_typography_for_font_pairing_queries(self):
|
||||
self.assertEqual(detect_domain("font pairing elegant serif body font"), "typography")
|
||||
|
||||
def test_router_prioritizes_chart_queries_over_generic_product_keywords(self):
|
||||
self.assertEqual(detect_domain("time series chart forecast"), "chart")
|
||||
|
||||
def test_hash_only_routes_color_for_a_valid_hex_literal(self):
|
||||
self.assertNotEqual(detect_domain("C# WPF desktop app"), "color")
|
||||
self.assertEqual(detect_domain("use #ff00aa as the accent"), "color")
|
||||
|
||||
def test_product_router_keeps_high_signal_service_aliases(self):
|
||||
self.assertEqual(detect_domain("beauty spa"), "product")
|
||||
self.assertEqual(detect_domain("salon booking"), "product")
|
||||
|
||||
def test_native_drag_intent_beats_generic_react_token(self):
|
||||
self.assertEqual(detect_domain("drag reorder react native"), "web")
|
||||
|
||||
def test_every_router_term_is_searchable_or_has_a_corpus_rewrite(self):
|
||||
for domain, keywords in core._domain_keywords().items():
|
||||
config = CSV_CONFIG[domain]
|
||||
path = core.DATA_DIR / config["file"]
|
||||
index = core._get_bm25(path, config["search_cols"], core._load_csv(path))
|
||||
vocabulary = set(index.vocabulary())
|
||||
for keyword in keywords:
|
||||
with self.subTest(domain=domain, keyword=keyword):
|
||||
searchable = bool(set(index.tokenize(keyword)) & vocabulary)
|
||||
explicitly_routing_only = keyword in core._DOMAIN_QUERY_REWRITES.get(domain, {})
|
||||
self.assertTrue(searchable or explicitly_routing_only)
|
||||
|
||||
|
||||
class TestPersistence(unittest.TestCase):
|
||||
def test_concurrent_non_force_persist_has_one_writer(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
search_script = SCRIPTS_DIR / "search.py"
|
||||
processes = [subprocess.Popen(
|
||||
[sys.executable, str(search_script), f"saas dashboard {index}",
|
||||
"--design-system", "--persist", "--project-name", "Race Probe",
|
||||
"--output-dir", tmp, "--json"],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
||||
) for index in range(8)]
|
||||
statuses = []
|
||||
for process in processes:
|
||||
stdout, stderr = process.communicate(timeout=30)
|
||||
self.assertEqual(process.returncode, 0, stderr)
|
||||
statuses.append(json.loads(stdout)["persistence"]["status"])
|
||||
|
||||
self.assertEqual(statuses.count("success"), 1)
|
||||
self.assertEqual(statuses.count("skipped_exists"), 7)
|
||||
|
||||
def test_persist_then_skip_then_force(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
result = generate_design_system("saas dashboard", "Test Project", persist=True, output_dir=tmp)
|
||||
@@ -106,6 +281,24 @@ class TestPersistence(unittest.TestCase):
|
||||
self.assertEqual(result2["persistence"]["status"], "skipped_exists")
|
||||
self.assertEqual(master.read_text(encoding="utf-8"), original_content)
|
||||
|
||||
# A new page override may be added without rewriting the existing Master.
|
||||
page_result = generate_design_system(
|
||||
"checkout form", "Test Project", persist=True, page="Checkout", output_dir=tmp
|
||||
)
|
||||
self.assertEqual(page_result["persistence"]["status"], "success")
|
||||
self.assertEqual(master.read_text(encoding="utf-8"), original_content)
|
||||
page_file = Path(tmp) / "design-system" / "test-project" / "pages" / "checkout.md"
|
||||
self.assertEqual(page_result["persistence"]["created_files"], [str(page_file)])
|
||||
self.assertTrue(page_file.exists())
|
||||
|
||||
# Existing page overrides are protected by the same default no-overwrite rule.
|
||||
page_content = page_file.read_text(encoding="utf-8")
|
||||
page_result2 = generate_design_system(
|
||||
"different checkout", "Test Project", persist=True, page="Checkout", output_dir=tmp
|
||||
)
|
||||
self.assertEqual(page_result2["persistence"]["status"], "skipped_exists")
|
||||
self.assertEqual(page_file.read_text(encoding="utf-8"), page_content)
|
||||
|
||||
# With force=True it must overwrite.
|
||||
result3 = generate_design_system("ecommerce luxury", "Test Project", persist=True, output_dir=tmp, force=True)
|
||||
self.assertEqual(result3["persistence"]["status"], "success")
|
||||
@@ -130,5 +323,21 @@ class TestReasoningMatch(unittest.TestCase):
|
||||
self.assertIsInstance(rule, dict)
|
||||
|
||||
|
||||
class TestDiagnosticsContracts(unittest.TestCase):
|
||||
def test_diagnostics_opt_in_is_additive_for_domain_search(self):
|
||||
baseline = search("minimalism", domain="style", max_results=1)
|
||||
diagnosed = search("minimalism", domain="style", max_results=1, diagnostics=True)
|
||||
self.assertEqual(set(baseline.keys()), set(diagnosed.keys()) - {"diagnostics"})
|
||||
self.assertIn("diagnostics", diagnosed)
|
||||
self.assertIn("top_score", diagnosed["diagnostics"])
|
||||
self.assertIn("query_rewrites", diagnosed["diagnostics"])
|
||||
|
||||
def test_diagnostics_opt_in_is_additive_for_stack_search(self):
|
||||
baseline = search_stack("performance", "react", max_results=1)
|
||||
diagnosed = search_stack("performance", "react", max_results=1, diagnostics=True)
|
||||
self.assertEqual(set(baseline.keys()), set(diagnosed.keys()) - {"diagnostics"})
|
||||
self.assertIn("diagnostics", diagnosed)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Semantic quality contracts for the core UI/UX datasets."""
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent
|
||||
DATA_DIR = SCRIPTS_DIR.parent / "data"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import search # noqa: E402
|
||||
from validate_data import ( # noqa: E402
|
||||
CHART_NON_COLOR_GUIDANCE,
|
||||
CHART_RISKS,
|
||||
CHART_TEXT_FALLBACK,
|
||||
COLOR_CONTRAST_PAIRS,
|
||||
CSS_IMPORT,
|
||||
ICON_CONTEXTS,
|
||||
ICON_ROLES,
|
||||
ICON_USAGE_REQUIREMENTS,
|
||||
WCAG_GRADE,
|
||||
_check_chart_contract,
|
||||
_check_icon_contract,
|
||||
_check_typography_contract,
|
||||
_configured_font_names,
|
||||
_font_families,
|
||||
_font_names,
|
||||
contrast_ratio,
|
||||
)
|
||||
|
||||
|
||||
def read_rows(name):
|
||||
with (DATA_DIR / name).open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
class TestSemanticColors(unittest.TestCase):
|
||||
def test_declared_text_and_ui_pairs_meet_role_thresholds(self):
|
||||
for row in read_rows("colors.csv"):
|
||||
for foreground, (background, role, minimum) in COLOR_CONTRAST_PAIRS.items():
|
||||
with self.subTest(
|
||||
product=row["Product Type"], pair=foreground, role=role):
|
||||
self.assertGreaterEqual(
|
||||
contrast_ratio(row[foreground], row[background]), minimum)
|
||||
|
||||
def test_destructive_tokens_are_not_success_green(self):
|
||||
for row in read_rows("colors.csv"):
|
||||
value = row["Destructive"].lstrip("#")
|
||||
red, green, blue = (int(value[index:index + 2], 16)
|
||||
for index in (0, 2, 4))
|
||||
with self.subTest(product=row["Product Type"]):
|
||||
self.assertFalse(green > red * 1.1 and green > blue * 1.1)
|
||||
|
||||
|
||||
class TestAccessibilityGuidance(unittest.TestCase):
|
||||
def test_wcag_22_topics_have_explicit_rows_and_are_retrievable(self):
|
||||
rows = read_rows("ux-guidelines.csv")
|
||||
issues = {row["Issue"] for row in rows}
|
||||
expected_platforms = {
|
||||
"Focus Not Obscured (Minimum)": "Web",
|
||||
"Focus Not Obscured (Enhanced)": "Web",
|
||||
"Focus Appearance": "Web",
|
||||
"Dragging Movements": "All",
|
||||
"Target Size (Minimum)": "Web",
|
||||
"Consistent Help": "All",
|
||||
"Redundant Entry": "All",
|
||||
"Accessible Authentication (Minimum)": "All",
|
||||
"Auto-Rotating Content Controls": "All",
|
||||
}
|
||||
self.assertTrue(expected_platforms.keys() <= issues)
|
||||
for issue, expected_platform in expected_platforms.items():
|
||||
with self.subTest(issue=issue):
|
||||
row = next(row for row in rows if row["Issue"] == issue)
|
||||
self.assertEqual(row["Platform"], expected_platform)
|
||||
self.assertIn(row["Severity"], {"Medium", "High", "Critical"})
|
||||
self.assertNotEqual(row["Do"], row["Description"])
|
||||
result = search(issue, domain="ux", max_results=3)
|
||||
self.assertTrue(any(row.get("Issue") == issue for row in result["results"]))
|
||||
|
||||
def test_native_and_web_target_sizes_remain_distinct(self):
|
||||
native = next(row for row in read_rows("app-interface.csv")
|
||||
if row["Issue"] == "Touch Target Size")
|
||||
web = next(row for row in read_rows("ux-guidelines.csv")
|
||||
if row["Issue"] == "Target Size (Minimum)")
|
||||
native_text = " ".join(native.values())
|
||||
web_text = " ".join(web.values())
|
||||
self.assertIn("44pt", native_text)
|
||||
self.assertIn("48dp", native_text)
|
||||
self.assertIn("24 CSS px", web_text)
|
||||
|
||||
def test_motion_recipes_offer_reduced_motion_or_user_control(self):
|
||||
for row in read_rows("motion.csv"):
|
||||
text = " ".join(row.values()).casefold()
|
||||
with self.subTest(row=row["No"], category=row["Category"]):
|
||||
self.assertTrue(
|
||||
"reduced-motion" in text or "user-controlled" in text,
|
||||
"every motion recipe needs an explicit opt-out",
|
||||
)
|
||||
|
||||
|
||||
class TestChartsTypographyAndIcons(unittest.TestCase):
|
||||
def test_mutated_accessibility_and_import_contracts_fail(self):
|
||||
mutations = []
|
||||
chart = dict(read_rows("charts.csv")[0])
|
||||
chart["A11y Fallback"] = "A visible table is available."
|
||||
mutations.append((_check_chart_contract, chart, "keyboard"))
|
||||
typography = dict(read_rows("typography.csv")[0])
|
||||
typography["CSS Import"] = (
|
||||
"@import url('https://fonts.googleapis.com/css2?family=Comic+Sans');"
|
||||
)
|
||||
mutations.append((_check_typography_contract, typography, "differ"))
|
||||
missing_weight = dict(read_rows("typography.csv")[0])
|
||||
missing_weight["Notes"] += " Recommended weight 900."
|
||||
mutations.append((_check_typography_contract, missing_weight, "weights"))
|
||||
icon = dict(read_rows("icons.csv")[0])
|
||||
icon["Import Code"] = "import { IconName } from '@phosphor-icons/react'"
|
||||
mutations.append((_check_icon_contract, icon, "import"))
|
||||
for checker, row, expected in mutations:
|
||||
with self.subTest(checker=checker.__name__):
|
||||
problems = []
|
||||
checker([row], problems)
|
||||
self.assertTrue(any(expected in problem for problem in problems))
|
||||
|
||||
def test_chart_risk_is_not_a_conformance_grade(self):
|
||||
for row in read_rows("charts.csv"):
|
||||
with self.subTest(data_type=row["Data Type"]):
|
||||
self.assertIn(row["Accessibility Risk"], CHART_RISKS)
|
||||
self.assertEqual(
|
||||
row["Accessibility Grade"],
|
||||
"deprecated: use Accessibility Risk",
|
||||
)
|
||||
text = " ".join((row["Accessibility Notes"], row["A11y Fallback"]))
|
||||
self.assertIsNone(WCAG_GRADE.search(text))
|
||||
self.assertIsNotNone(CHART_TEXT_FALLBACK.search(text.casefold()))
|
||||
self.assertIsNotNone(
|
||||
CHART_NON_COLOR_GUIDANCE.search(text.casefold())
|
||||
)
|
||||
self.assertIn("keyboard", text.casefold())
|
||||
|
||||
def test_named_fonts_match_google_import_css_import_and_tailwind_config(self):
|
||||
for row in read_rows("typography.csv"):
|
||||
url_families = _font_families(row["Google Fonts URL"])
|
||||
families = _font_names(url_families)
|
||||
configured = _configured_font_names(row["Tailwind Config"])
|
||||
named = {row["Heading Font"], row["Body Font"]}
|
||||
with self.subTest(pairing=row["Font Pairing Name"]):
|
||||
self.assertTrue(named <= families)
|
||||
self.assertTrue(named <= configured)
|
||||
match = CSS_IMPORT.fullmatch(row["CSS Import"])
|
||||
self.assertIsNotNone(match)
|
||||
self.assertEqual(
|
||||
sorted(url_families),
|
||||
sorted(_font_families(match.group(2))),
|
||||
)
|
||||
|
||||
def test_icon_semantics_are_explicit_and_imports_are_concrete(self):
|
||||
for row in read_rows("icons.csv"):
|
||||
with self.subTest(icon=row["Icon Name"]):
|
||||
self.assertIn(row["Semantic Role"], ICON_ROLES)
|
||||
self.assertEqual(set(row["Allowed Contexts"].split("|")), ICON_CONTEXTS)
|
||||
self.assertNotRegex(row["Usage"], r"[\u3400-\u9fff]")
|
||||
self.assertNotIn("IconName", row["Import Code"])
|
||||
for requirement in ICON_USAGE_REQUIREMENTS:
|
||||
self.assertRegex(row["Usage"].casefold(), requirement)
|
||||
|
||||
def test_natural_accessibility_queries_are_retrievable(self):
|
||||
cases = {
|
||||
"chart": ("keyboard accessible chart", "keyboard"),
|
||||
"landing": ("accessible drag interaction", "keyboard controls"),
|
||||
"icons": ("decorative icon aria hidden", "aria-hidden"),
|
||||
"gsap": ("stop animation offscreen", "visibility"),
|
||||
"ux": ("error summary validation", "error summary"),
|
||||
}
|
||||
for domain, (query, expected) in cases.items():
|
||||
with self.subTest(domain=domain, query=query):
|
||||
result = search(query, domain=domain, max_results=3)
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertIn(
|
||||
expected.casefold(),
|
||||
" ".join(str(value) for value in result["results"][0].values()).casefold(),
|
||||
)
|
||||
|
||||
|
||||
class TestCurrentReactGuidance(unittest.TestCase):
|
||||
def test_effect_event_is_scoped_and_community_use_latest_is_removed(self):
|
||||
rows = read_rows("react-performance.csv")
|
||||
effect_event = next(row for row in rows if row["Issue"] == "Effect Events")
|
||||
text = " ".join(effect_event.values()).casefold()
|
||||
self.assertIn("inside effects", text)
|
||||
self.assertIn("dependencies", text)
|
||||
self.assertFalse(any("uselatest" in " ".join(row.values()).casefold()
|
||||
for row in rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,421 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross-file semantic contracts for curated design data."""
|
||||
|
||||
import copy
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent
|
||||
DATA_DIR = SCRIPTS_DIR.parent / "data"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import AVAILABLE_STACKS, STACK_CONFIG # noqa: E402
|
||||
from design_system import DesignSystemGenerator # noqa: E402
|
||||
from reasoning_contract import apply_decision_rules, parse_decision_rules # noqa: E402
|
||||
import validate_data # noqa: E402
|
||||
from validate_data import _check_reasoning_contract # noqa: E402
|
||||
|
||||
|
||||
def read_rows(name):
|
||||
with (DATA_DIR / name).open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
def split_values(value, delimiter):
|
||||
return [part.strip() for part in value.split(delimiter) if part.strip()]
|
||||
|
||||
|
||||
def style_identities(row):
|
||||
return [
|
||||
row["Style ID"], row["Style Category"],
|
||||
*split_values(row["Aliases"], "|"),
|
||||
]
|
||||
|
||||
|
||||
class TestStyleIdentityContract(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.styles = read_rows("styles.csv")
|
||||
|
||||
def test_ids_aliases_status_and_parents_are_unambiguous(self):
|
||||
ids = {row["Style ID"] for row in self.styles}
|
||||
self.assertEqual(len(ids), len(self.styles))
|
||||
aliases = {}
|
||||
for row in self.styles:
|
||||
style_id = row["Style ID"]
|
||||
self.assertRegex(style_id, r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
self.assertIn(row["Status"], {"active", "supplemental", "deprecated"})
|
||||
parent = row["Parent Style ID"]
|
||||
if parent:
|
||||
self.assertIn(parent, ids)
|
||||
self.assertNotEqual(parent, style_id)
|
||||
if row["Status"] == "supplemental":
|
||||
self.assertTrue(parent)
|
||||
if row["Status"] == "deprecated":
|
||||
has_redirect = bool(row["Replacement Domain"] and row["Replacement ID"])
|
||||
self.assertNotEqual(bool(parent), has_redirect)
|
||||
for alias in split_values(row["Aliases"], "|"):
|
||||
self.assertNotIn(alias.casefold(), aliases)
|
||||
aliases[alias.casefold()] = style_id
|
||||
|
||||
def test_every_product_and_reasoning_style_reference_resolves(self):
|
||||
lookup = {}
|
||||
for row in self.styles:
|
||||
lookup.update(
|
||||
(identity.casefold(), row["Style ID"])
|
||||
for identity in style_identities(row)
|
||||
)
|
||||
|
||||
references = []
|
||||
for row in read_rows("products.csv"):
|
||||
references.extend(split_values(row["Primary Style Recommendation"], "+"))
|
||||
references.extend(split_values(row["Secondary Styles"], ","))
|
||||
for row in read_rows("ui-reasoning.csv"):
|
||||
references.extend(split_values(row["Style_Priority"], "+"))
|
||||
unresolved = sorted({
|
||||
reference for reference in references
|
||||
if reference.casefold() not in lookup
|
||||
})
|
||||
self.assertEqual([], unresolved)
|
||||
|
||||
|
||||
class TestReasoningContract(unittest.TestCase):
|
||||
def test_known_product_sets_match_exactly(self):
|
||||
product_rows = read_rows("products.csv")
|
||||
color_rows = read_rows("colors.csv")
|
||||
reasoning_rows = read_rows("ui-reasoning.csv")
|
||||
self.assertEqual([192, 192, 192], [
|
||||
len(product_rows), len(color_rows), len(reasoning_rows)])
|
||||
products = {row["Product Type"] for row in product_rows}
|
||||
colors = {row["Product Type"] for row in color_rows}
|
||||
reasoning = {row["UI_Category"] for row in reasoning_rows}
|
||||
self.assertEqual(products, colors)
|
||||
self.assertEqual(products, reasoning)
|
||||
self.assertEqual(len(products), 192)
|
||||
|
||||
def test_decision_rules_use_closed_array_grammar(self):
|
||||
for row in read_rows("ui-reasoning.csv"):
|
||||
with self.subTest(category=row["UI_Category"]):
|
||||
parsed = parse_decision_rules(row["Decision_Rules"])
|
||||
self.assertTrue(all(isinstance(actions, list) for actions in parsed.values()))
|
||||
|
||||
def test_duplicate_unknown_keys_and_unknown_actions_fail_closed(self):
|
||||
invalid = (
|
||||
'{"must_have":["constraint:first"],"must_have":["constraint:second"]}',
|
||||
'{"if_not_supported":["constraint:test"]}',
|
||||
'{"must_have":["execute:arbitrary"]}',
|
||||
'{"must_have":[["constraint:nested"]]}',
|
||||
'{"must_have":[{"constraint":"nested"}]}',
|
||||
)
|
||||
for raw in invalid:
|
||||
with self.subTest(raw=raw), self.assertRaises(ValueError):
|
||||
parse_decision_rules(raw)
|
||||
|
||||
def test_must_have_and_explicit_signals_are_applied_and_reported(self):
|
||||
rules = parse_decision_rules(
|
||||
'{"must_have":["constraint:keyboard-navigation"],'
|
||||
'"if_mobile":["constraint:optimize-touch-targets"]}')
|
||||
desktop = apply_decision_rules(rules, "accessible government portal")
|
||||
mobile = apply_decision_rules(rules, "accessible mobile government portal")
|
||||
self.assertEqual(desktop["constraints"], ["keyboard-navigation"])
|
||||
self.assertEqual(
|
||||
mobile["constraints"], ["keyboard-navigation", "optimize-touch-targets"])
|
||||
self.assertEqual(
|
||||
[item["condition"] for item in mobile["activated"]],
|
||||
["must_have", "if_mobile"],
|
||||
)
|
||||
|
||||
def test_generator_matches_reasoning_exactly_and_defaults_only_for_unknown(self):
|
||||
generator = DesignSystemGenerator()
|
||||
categories = [row["Product Type"] for row in read_rows("products.csv")]
|
||||
for category in categories:
|
||||
with self.subTest(category=category):
|
||||
self.assertEqual(generator._find_reasoning_rule(category)["UI_Category"], category)
|
||||
self.assertEqual(generator._find_reasoning_rule("Government"), {})
|
||||
self.assertTrue(generator._apply_reasoning("External Unknown", "unknown")["is_default"])
|
||||
|
||||
def test_reasoning_patterns_reference_landing_identities(self):
|
||||
patterns = set()
|
||||
for row in read_rows("landing.csv"):
|
||||
patterns.add(row["Pattern Name"])
|
||||
patterns.update(alias for alias in row["Aliases"].split("|") if alias)
|
||||
reasoning = read_rows("ui-reasoning.csv")
|
||||
self.assertEqual(192, len(reasoning))
|
||||
for row in reasoning:
|
||||
with self.subTest(category=row["UI_Category"]):
|
||||
self.assertIn(row["Recommended_Pattern"], patterns)
|
||||
|
||||
def test_every_known_product_generates_a_traceable_landing_pattern(self):
|
||||
generator = DesignSystemGenerator()
|
||||
patterns = {row["Pattern Name"] for row in read_rows("landing.csv")}
|
||||
for category in (row["Product Type"] for row in read_rows("products.csv")):
|
||||
with self.subTest(category=category):
|
||||
result = generator.generate(category)
|
||||
self.assertIn(result["source_identities"]["landing"], patterns)
|
||||
|
||||
def test_representative_new_products_generate_traceable_sources(self):
|
||||
generator = DesignSystemGenerator()
|
||||
styles = {row["Style ID"] for row in read_rows("styles.csv")}
|
||||
colors = {row["Product Type"] for row in read_rows("colors.csv")}
|
||||
typography = {row["Font Pairing Name"] for row in read_rows("typography.csv")}
|
||||
patterns = {row["Pattern Name"] for row in read_rows("landing.csv")}
|
||||
cases = {
|
||||
"government grant portal accessible trustworthy": "Grant / Funding Portal",
|
||||
"API developer portal documentation": "API Developer Portal",
|
||||
"academic journal scholarly publishing accessible": "Academic Journal / Scholarly Publishing",
|
||||
"patient portal mobile secure": "Patient Portal / Health Records",
|
||||
"status page outage monitoring": "Status Page / Incident Management",
|
||||
}
|
||||
for query, category in cases.items():
|
||||
with self.subTest(query=query):
|
||||
result = generator.generate(query)
|
||||
sources = result["source_identities"]
|
||||
self.assertEqual(category, result["category"])
|
||||
self.assertFalse(result["reasoning_default"])
|
||||
self.assertEqual(category, sources["product"])
|
||||
self.assertEqual(category, sources["reasoning"])
|
||||
self.assertIn(sources["style"], styles)
|
||||
self.assertIn(sources["color"], colors)
|
||||
self.assertIn(sources["typography"], typography)
|
||||
self.assertIn(sources["landing"], patterns)
|
||||
|
||||
def test_constraints_reach_domain_queries(self):
|
||||
generator = DesignSystemGenerator()
|
||||
calls = []
|
||||
|
||||
def capture(query, domain, max_results):
|
||||
calls.append((domain, query))
|
||||
return {"domain": domain, "count": 0, "results": []}
|
||||
|
||||
reasoning = {
|
||||
"pattern": "Unmapped Pattern",
|
||||
"color_mood": "Trustworthy",
|
||||
"typography_mood": "Readable",
|
||||
"constraints": ["keyboard-navigation", "touch-targets"],
|
||||
}
|
||||
with patch("design_system.search", side_effect=capture):
|
||||
generator._multi_domain_search(
|
||||
"public portal", "Government Portal", reasoning, ["Minimalism"])
|
||||
queried = {domain: query for domain, query in calls}
|
||||
for domain in ("style", "color", "typography", "landing"):
|
||||
with self.subTest(domain=domain):
|
||||
self.assertIn("keyboard navigation", queried[domain])
|
||||
|
||||
def test_canonical_style_priority_is_not_limited_to_bm25_top_three(self):
|
||||
generator = DesignSystemGenerator()
|
||||
unrelated = [
|
||||
generator._resolve_style("Kinetic Brutalism (Mobile)"),
|
||||
generator._resolve_style("Glassmorphism"),
|
||||
]
|
||||
selected = generator._select_best_match(unrelated, ["Brutalism"])
|
||||
self.assertEqual("brutalism", selected["Style ID"])
|
||||
|
||||
def test_duplicate_semantic_reasoning_labels_fail_validation(self):
|
||||
product = {"Product Type": "Duplicate"}
|
||||
color = {"Product Type": "Duplicate"}
|
||||
reasoning = {
|
||||
"UI_Category": "Duplicate", "Decision_Rules": "{}", "Confidence": ""
|
||||
}
|
||||
problems = []
|
||||
_check_reasoning_contract(
|
||||
[product, dict(product)], [color, dict(color)],
|
||||
[reasoning, dict(reasoning)], set(), set(), problems,
|
||||
)
|
||||
self.assertTrue(any("duplicate" in problem.lower() for problem in problems))
|
||||
|
||||
def test_every_exact_product_label_resolves_to_itself(self):
|
||||
generator = DesignSystemGenerator()
|
||||
for row in read_rows("products.csv"):
|
||||
category = row["Product Type"]
|
||||
with self.subTest(category=category):
|
||||
result = generator.generate(category)
|
||||
reasoning = generator._apply_reasoning(category, category)
|
||||
expected = [
|
||||
generator._resolve_style(priority).get("Style ID")
|
||||
for priority in reasoning["style_priority"]
|
||||
]
|
||||
expected = [style_id for style_id in expected if style_id]
|
||||
self.assertEqual(category, result["category"])
|
||||
self.assertFalse(result["reasoning_default"])
|
||||
self.assertTrue(expected)
|
||||
self.assertEqual(expected[0], result["style"]["id"])
|
||||
|
||||
def test_style_aliases_have_one_exact_owner(self):
|
||||
generator = DesignSystemGenerator()
|
||||
self.assertEqual(
|
||||
generator._resolve_style("Minimalism")["Style ID"],
|
||||
"minimalism-and-swiss-style",
|
||||
)
|
||||
self.assertEqual(generator._resolve_style("Clean Science"), {})
|
||||
self.assertEqual(
|
||||
generator._resolve_style("Holographic/HUD")["Style ID"],
|
||||
"hud-sci-fi-fui",
|
||||
)
|
||||
|
||||
|
||||
class TestLandingAndStackContract(unittest.TestCase):
|
||||
def test_landing_sections_use_one_delimiter(self):
|
||||
for row in read_rows("landing.csv"):
|
||||
with self.subTest(pattern=row["Pattern Name"]):
|
||||
sections = row["Section Order"].split(" > ")
|
||||
self.assertGreaterEqual(len(sections), 2)
|
||||
self.assertTrue(all(section.strip() for section in sections))
|
||||
self.assertFalse(any(re.match(r"^\d+\.\s", section) for section in sections))
|
||||
|
||||
def test_stack_schema_is_additive_and_uniform(self):
|
||||
for stack in AVAILABLE_STACKS:
|
||||
path = DATA_DIR / STACK_CONFIG[stack]["file"]
|
||||
with path.open(encoding="utf-8", newline="") as handle:
|
||||
reader = csv.DictReader(handle)
|
||||
self.assertTrue({"Applies To", "Status", "Verified At"} <= set(reader.fieldnames or []))
|
||||
for row in reader:
|
||||
self.assertIn(row["Status"], {"active", "supplemental", "deprecated", "unverified"})
|
||||
|
||||
def test_provenance_sidecar_has_stable_shape(self):
|
||||
payload = json.loads((DATA_DIR / "data-provenance.json").read_text(encoding="utf-8"))
|
||||
self.assertEqual(payload["schemaVersion"], 1)
|
||||
self.assertIsInstance(payload["records"], list)
|
||||
for record in payload["records"]:
|
||||
self.assertTrue({"entityKind", "entityId", "sourceFile", "status", "verifiedAt", "sources"} <= set(record))
|
||||
self.assertIsInstance(record["sources"], list)
|
||||
source_types = {source.get("type") for source in record["sources"]}
|
||||
if source_types <= {"derived"}:
|
||||
self.assertEqual("needs-review", record["sla"])
|
||||
|
||||
def test_provenance_rejects_bad_shapes_enums_and_hosts_without_crashing(self):
|
||||
canonical = json.loads(
|
||||
(DATA_DIR / "data-provenance.json").read_text(encoding="utf-8")
|
||||
)
|
||||
cases = [[], None, "invalid"]
|
||||
malformed_record = copy.deepcopy(canonical)
|
||||
malformed_record["records"].append(None)
|
||||
cases.append(malformed_record)
|
||||
malformed_source = copy.deepcopy(canonical)
|
||||
malformed_source["records"][0]["sources"] = [None]
|
||||
cases.append(malformed_source)
|
||||
unapproved_source = copy.deepcopy(canonical)
|
||||
official = next(
|
||||
record for record in unapproved_source["records"]
|
||||
if any(source.get("type") == "official" for source in record["sources"])
|
||||
)
|
||||
official["sources"] = [
|
||||
{"type": "official", "ref": "https://evil.example/fake"}
|
||||
]
|
||||
cases.append(unapproved_source)
|
||||
invalid_enums = copy.deepcopy(canonical)
|
||||
invalid_enums["records"][0].update(
|
||||
status="invented", sla="whenever", confidence=float("nan")
|
||||
)
|
||||
cases.append(invalid_enums)
|
||||
|
||||
reasoning, styles = read_rows("ui-reasoning.csv"), read_rows("styles.csv")
|
||||
for index, payload in enumerate(cases):
|
||||
with self.subTest(case=index), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / "data-provenance.json").write_text(
|
||||
json.dumps(payload), encoding="utf-8"
|
||||
)
|
||||
problems = []
|
||||
with patch.object(validate_data, "DATA_DIR", root):
|
||||
validate_data._check_provenance(reasoning, styles, problems)
|
||||
self.assertTrue(problems)
|
||||
|
||||
def test_dataset_provenance_scope_binds_real_rows_and_fields(self):
|
||||
problems = []
|
||||
valid = validate_data._valid_dataset_source_key(
|
||||
"colors.csv", {"Scope": "No 1-192; Notes field"},
|
||||
("dataset-contract", "valid"), problems,
|
||||
)
|
||||
self.assertTrue(valid)
|
||||
self.assertEqual([], problems)
|
||||
stack_problems = []
|
||||
self.assertTrue(validate_data._valid_dataset_source_key(
|
||||
"stacks/html-tailwind.csv", {"Scope": "No 57-59; Guideline field"},
|
||||
("dataset-contract", "valid-stack"), stack_problems,
|
||||
))
|
||||
self.assertEqual([], stack_problems)
|
||||
for source_file, source_key in (
|
||||
("unknown.csv", {"Scope": "No 1; Notes field"}),
|
||||
("colors.csv", {"Scope": "No 999; Notes field"}),
|
||||
("colors.csv", {"Scope": "No 1; Invented Field"}),
|
||||
):
|
||||
with self.subTest(source_file=source_file, source_key=source_key):
|
||||
problems = []
|
||||
self.assertFalse(validate_data._valid_dataset_source_key(
|
||||
source_file, source_key, ("dataset-contract", "bad"), problems
|
||||
))
|
||||
self.assertTrue(problems)
|
||||
|
||||
|
||||
class TestGeneratedCatalogContract(unittest.TestCase):
|
||||
def load_json(self, name):
|
||||
return json.loads((DATA_DIR / name).read_text(encoding="utf-8"))
|
||||
|
||||
def test_canonical_catalogs_and_provenance_are_release_ready(self):
|
||||
problems = validate_data.validate()
|
||||
self.assertEqual([], [problem for problem in problems if "catalog" in problem])
|
||||
|
||||
def test_font_license_and_typography_drift_fail_closed(self):
|
||||
fonts = read_rows("google-fonts.csv")
|
||||
licenses = self.load_json("google-font-licenses.json")
|
||||
licenses["families"][0]["license"] = "UNKNOWN"
|
||||
problems = []
|
||||
validate_data._check_font_catalog(
|
||||
fonts, licenses, read_rows("typography.csv"), problems
|
||||
)
|
||||
self.assertTrue(any("invalid active family" in problem for problem in problems))
|
||||
|
||||
missing_font = copy.deepcopy(read_rows("typography.csv"))
|
||||
missing_font[0]["Google Fonts URL"] = (
|
||||
"https://fonts.googleapis.com/css2?family=Invented+Sans:wght@400"
|
||||
)
|
||||
problems = []
|
||||
validate_data._check_font_catalog(
|
||||
fonts, self.load_json("google-font-licenses.json"), missing_font, problems
|
||||
)
|
||||
self.assertTrue(any("absent from approved catalog" in problem for problem in problems))
|
||||
|
||||
def test_font_source_revision_and_exclusion_policy_fail_closed(self):
|
||||
fonts = read_rows("google-fonts.csv")
|
||||
typography = read_rows("typography.csv")
|
||||
licenses = self.load_json("google-font-licenses.json")
|
||||
licenses["excludedFamilies"][0]["source"] = "https://github.com/google/fonts"
|
||||
problems = []
|
||||
validate_data._check_font_catalog(fonts, licenses, typography, problems)
|
||||
self.assertFalse(any("invalid exclusion" in problem for problem in problems))
|
||||
|
||||
licenses["excludedFamilies"][0]["source"] = "https://github.com/other/fonts"
|
||||
licenses["source"]["revision"] = "main"
|
||||
problems = []
|
||||
validate_data._check_font_catalog(fonts, licenses, typography, problems)
|
||||
self.assertTrue(any("invalid exclusion" in problem for problem in problems))
|
||||
self.assertTrue(any("invalid source revision" in problem for problem in problems))
|
||||
|
||||
def test_curated_icon_and_summary_drift_fail_closed(self):
|
||||
manifest = self.load_json("phosphor-icons-upstream.json")
|
||||
manifest["icons"][0]["clientImport"] = (
|
||||
'import { Wrong } from "@phosphor-icons/react"'
|
||||
)
|
||||
problems = []
|
||||
validate_data._check_phosphor_catalog(read_rows("icons.csv"), manifest, problems)
|
||||
self.assertTrue(any("invalid identity or imports" in problem for problem in problems))
|
||||
|
||||
summary = self.load_json("catalog-summary.json")
|
||||
summary["counts"]["googleFonts"] -= 1
|
||||
problems = []
|
||||
validate_data._check_catalog_summary(
|
||||
summary,
|
||||
self.load_json("google-font-licenses.json"),
|
||||
self.load_json("phosphor-icons-upstream.json"),
|
||||
problems,
|
||||
)
|
||||
self.assertTrue(any("stale count for googleFonts" in problem for problem in problems))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -25,6 +25,7 @@ sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from design_system import ( # noqa: E402
|
||||
_filter_anti_patterns_for_mode,
|
||||
_contrast_ratio,
|
||||
_palette_is_dark,
|
||||
_query_wants_dark,
|
||||
_relative_luminance,
|
||||
@@ -106,6 +107,15 @@ class TestPaletteSelection(unittest.TestCase):
|
||||
def test_empty_results(self):
|
||||
self.assertEqual(_select_palette_for_mode([], "dark"), {})
|
||||
|
||||
def test_category_identity_wins_over_unrelated_dark_palette(self):
|
||||
chosen = _select_palette_for_mode(
|
||||
[LIGHT_PALETTE, DARK_PALETTE], "dark", "SaaS")
|
||||
self.assertEqual("SaaS", chosen["Product Type"])
|
||||
self.assertEqual("derived-dark", chosen["_mode_derivation"])
|
||||
self.assertTrue(_palette_is_dark(chosen))
|
||||
self.assertGreaterEqual(
|
||||
_contrast_ratio(chosen["Ring"], chosen["Background"]), 3.0)
|
||||
|
||||
|
||||
class TestAntiPatternGating(unittest.TestCase):
|
||||
def test_dark_clause_dropped_others_kept(self):
|
||||
@@ -138,6 +148,26 @@ class TestEndToEndCoherence(unittest.TestCase):
|
||||
"dark-mode query returned a light background: {}".format(background),
|
||||
)
|
||||
|
||||
def test_generator_exports_every_semantic_foreground_pair(self):
|
||||
colors = DesignSystemGenerator().generate("SaaS dashboard")["colors"]
|
||||
pairs = (
|
||||
("on_primary", "primary"),
|
||||
("on_secondary", "secondary"),
|
||||
("on_accent", "accent"),
|
||||
("foreground", "background"),
|
||||
("card_foreground", "card"),
|
||||
("muted_foreground", "muted"),
|
||||
("on_destructive", "destructive"),
|
||||
)
|
||||
for foreground, background in pairs:
|
||||
with self.subTest(pair=foreground):
|
||||
self.assertTrue(colors[foreground])
|
||||
self.assertTrue(colors[background])
|
||||
self.assertGreaterEqual(
|
||||
_contrast_ratio(colors[foreground], colors[background]), 4.5
|
||||
)
|
||||
self.assertEqual(colors["on_cta"], colors["on_accent"])
|
||||
|
||||
def test_dark_query_foreground_is_lighter_than_background(self):
|
||||
ds = DesignSystemGenerator().generate(self.QUERY)
|
||||
background = _relative_luminance(ds["colors"]["background"])
|
||||
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freshness and migration contracts for native, desktop, and 3D stacks."""
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import (DATA_DIR, STACK_CONFIG, STACK_CURRENT_APPLICABILITY,
|
||||
search_stack) # noqa: E402
|
||||
from validate_data import STACK_OFFICIAL_HOSTS # noqa: E402
|
||||
|
||||
STACKS = {
|
||||
"react-native", "flutter", "swiftui", "jetpack-compose", "avalonia",
|
||||
"uwp", "winui", "wpf", "uno", "javafx", "threejs", "laravel",
|
||||
}
|
||||
|
||||
|
||||
def _rows(stack):
|
||||
path = DATA_DIR / STACK_CONFIG[stack]["file"]
|
||||
with path.open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
class TestNativeDesktopStackFreshness(unittest.TestCase):
|
||||
def test_rows_have_final_freshness_metadata(self):
|
||||
for stack in STACKS:
|
||||
for row in _rows(stack):
|
||||
with self.subTest(stack=stack, row=row["No"]):
|
||||
expected = "deprecated" if stack == "uwp" else "active"
|
||||
self.assertEqual(row["Status"], expected)
|
||||
self.assertTrue(row["Applies To"].startswith(
|
||||
STACK_CURRENT_APPLICABILITY[stack]))
|
||||
self.assertRegex(row["Verified At"], r"^\d{4}-\d{2}-\d{2}$")
|
||||
self.assertEqual("legacy" in row["Applies To"], stack == "uwp")
|
||||
|
||||
def test_high_impact_rows_use_official_sources(self):
|
||||
for stack in STACKS:
|
||||
for row in _rows(stack):
|
||||
if row["Severity"] not in {"Critical", "High"}:
|
||||
continue
|
||||
with self.subTest(stack=stack, row=row["No"]):
|
||||
parsed = urlsplit(row["Docs URL"])
|
||||
self.assertEqual(parsed.scheme, "https")
|
||||
self.assertIn(parsed.hostname, STACK_OFFICIAL_HOSTS[stack])
|
||||
|
||||
def test_current_mobile_contracts(self):
|
||||
cases = {
|
||||
("react-native", "Hermes bundled default engine"): "hermes",
|
||||
("flutter", "predictive back result callback"): "onpopinvokedwithresult",
|
||||
("flutter", "accessible nonlinear text scaling"): "textscaler",
|
||||
("swiftui", "multicolumn navigation sidebar detail"): "navigationsplitview",
|
||||
}
|
||||
for (stack, query), expected in cases.items():
|
||||
with self.subTest(stack=stack):
|
||||
result = search_stack(query, stack, max_results=1)
|
||||
self.assertEqual(result["count"], 1)
|
||||
recommended = " ".join(
|
||||
result["results"][0][field]
|
||||
for field in ("Guideline", "Do", "Code Good")
|
||||
).casefold()
|
||||
self.assertIn(expected, recommended)
|
||||
|
||||
def test_windows_current_and_maintenance_lanes_are_visible(self):
|
||||
current = search_stack("new Windows desktop app Windows App SDK", "winui")
|
||||
legacy = search_stack("UWP maintenance x:Bind migration", "uwp")
|
||||
self.assertGreater(current["count"], 0)
|
||||
self.assertGreater(legacy["count"], 0)
|
||||
self.assertEqual({row["Status"] for row in current["results"]}, {"active"})
|
||||
self.assertEqual({row["Status"] for row in legacy["results"]}, {"deprecated"})
|
||||
self.assertTrue(any("winui" in " ".join(row.values()).casefold()
|
||||
for row in legacy["results"]))
|
||||
successor = search_stack(
|
||||
"which Windows UI framework should a brand new app choose instead of legacy UWP",
|
||||
"uwp", max_results=1,
|
||||
)
|
||||
self.assertEqual("Prefer WinUI 3 for new projects", successor["results"][0]["Guideline"])
|
||||
|
||||
def test_old_version_without_curated_rows_abstains(self):
|
||||
cases = {
|
||||
"react-native": "React Native 0.75 Hermes",
|
||||
"flutter": "Flutter SDK 3.22 back navigation",
|
||||
"swiftui": "iOS 15 SwiftUI navigation",
|
||||
"avalonia": "Avalonia UI v11 storage picker",
|
||||
"winui": "WinUI SDK 2 desktop app",
|
||||
"javafx": "JavaFX SDK 21 table view",
|
||||
"threejs": "Three.js r128 OrbitControls",
|
||||
"laravel": "Laravel 12 validation",
|
||||
}
|
||||
for stack, query in cases.items():
|
||||
with self.subTest(stack=stack):
|
||||
self.assertEqual(search_stack(query, stack)["count"], 0)
|
||||
|
||||
def test_migration_intent_returns_current_replacements(self):
|
||||
cases = {
|
||||
"flutter": "replace deprecated WillPopScope with current Flutter API",
|
||||
"react-native": "upgrade legacy Hermes configuration",
|
||||
"threejs": "replace deprecated outputEncoding with current color space",
|
||||
}
|
||||
for stack, query in cases.items():
|
||||
with self.subTest(stack=stack):
|
||||
result = search_stack(query, stack)
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual({row["Status"] for row in result["results"]}, {"active"})
|
||||
|
||||
versioned_cases = {
|
||||
"react-native": "upgrade React Native 0.75 to React Native 0.86 Hermes",
|
||||
"flutter": "upgrade Flutter 3.22 to Flutter 3.44 predictive back",
|
||||
"threejs": "upgrade Three.js r128 to r185 outputColorSpace",
|
||||
}
|
||||
for stack, query in versioned_cases.items():
|
||||
with self.subTest(stack=stack, query=query):
|
||||
result = search_stack(query, stack)
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual({row["Status"] for row in result["results"]}, {"active"})
|
||||
|
||||
def test_standalone_current_and_deprecated_identifiers_resolve_replacement(self):
|
||||
cases = {
|
||||
("flutter", "onPopInvokedWithResult"): "PopScope",
|
||||
("flutter", "TextScaler"): "large fonts",
|
||||
("threejs", "outputColorSpace"): "Color Space",
|
||||
("threejs", "outputEncoding"): "Color Space",
|
||||
}
|
||||
for (stack, query), guideline in cases.items():
|
||||
with self.subTest(stack=stack, query=query):
|
||||
result = search_stack(query, stack, max_results=1)
|
||||
self.assertEqual(result["count"], 1)
|
||||
self.assertIn(guideline.casefold(), result["results"][0]["Guideline"].casefold())
|
||||
|
||||
def test_current_threejs_uses_supported_module_and_color_apis(self):
|
||||
cases = {
|
||||
"Three.js current OrbitControls addon import": "three/addons/",
|
||||
"Three.js current renderer color space": "outputcolorspace",
|
||||
}
|
||||
for query, expected in cases.items():
|
||||
with self.subTest(query=query):
|
||||
result = search_stack(query, "threejs", max_results=1)
|
||||
self.assertEqual(result["count"], 1)
|
||||
recommended = " ".join(
|
||||
result["results"][0][field]
|
||||
for field in ("Guideline", "Do", "Code Good")
|
||||
).casefold()
|
||||
self.assertIn(expected, recommended)
|
||||
|
||||
def test_deprecated_symbols_are_not_recommended_by_current_rows(self):
|
||||
forbidden = {
|
||||
"react-native": ("hermes_enabled",),
|
||||
"flutter": ("willpopscope", "onpopinvoked:", "textscalefactor"),
|
||||
"swiftui": ("navigationview", "presentationmode"),
|
||||
"winui": ("dispatcher.runasync", "system.windows"),
|
||||
"uno": ("system.windows", 'requestedtheme="default"'),
|
||||
"javafx": ("fxpermission", "javadoc/21"),
|
||||
"threejs": (
|
||||
"r128", "three.orbitcontrols", "outputencoding",
|
||||
"three.srgbencoding", "examples/js/controls",
|
||||
),
|
||||
}
|
||||
for stack, tokens in forbidden.items():
|
||||
for row in _rows(stack):
|
||||
if row["Status"] != "active":
|
||||
continue
|
||||
recommended = " ".join(
|
||||
row[field] for field in ("Guideline", "Do", "Code Good")
|
||||
).casefold()
|
||||
for token in tokens:
|
||||
with self.subTest(stack=stack, row=row["No"], token=token):
|
||||
self.assertNotIn(token, recommended)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unit tests for metric math and relevance fixture validation."""
|
||||
|
||||
import importlib.util
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = next(parent for parent in Path(__file__).resolve().parents
|
||||
if (parent / "scripts/evaluate-relevance.py").exists())
|
||||
MODULE_PATH = ROOT / "scripts/evaluate-relevance.py"
|
||||
SPEC = importlib.util.spec_from_file_location("evaluate_relevance", MODULE_PATH)
|
||||
evaluator = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(evaluator)
|
||||
|
||||
|
||||
class TestMetricMath(unittest.TestCase):
|
||||
def test_precision_counts_missing_ranks_as_non_relevant(self):
|
||||
self.assertEqual(evaluator.precision_at_k([2], 1), 1.0)
|
||||
self.assertAlmostEqual(evaluator.precision_at_k([2], 3), 1 / 3)
|
||||
self.assertEqual(evaluator.precision_at_k([], 3), 0.0)
|
||||
self.assertEqual(evaluator.precision_at_k([2], 0), 0.0)
|
||||
|
||||
def test_reciprocal_rank_stops_at_k(self):
|
||||
self.assertEqual(evaluator.reciprocal_rank([0, 2, 0]), 0.5)
|
||||
self.assertEqual(evaluator.reciprocal_rank([0, 0, 0, 2]), 0.0)
|
||||
self.assertEqual(evaluator.reciprocal_rank([]), 0.0)
|
||||
|
||||
def test_ndcg_uses_graded_gain_and_handles_empty_ideal(self):
|
||||
self.assertEqual(evaluator.ndcg_at_k([2, 1], [2, 1]), 1.0)
|
||||
self.assertLess(evaluator.ndcg_at_k([1, 2], [2, 1]), 1.0)
|
||||
self.assertEqual(evaluator.ndcg_at_k([], [], 3), 0.0)
|
||||
|
||||
def test_result_grades_match_identity_subsets(self):
|
||||
results = [
|
||||
{"Category": "State", "Guideline": "Use useState", "Severity": "Medium"},
|
||||
{"Category": "State", "Guideline": "Use useReducer", "Severity": "Medium"},
|
||||
]
|
||||
judgments = [
|
||||
{"identity": {"Guideline": "Use useReducer"}, "grade": 2},
|
||||
{"identity": {"Category": "State"}, "grade": 1},
|
||||
]
|
||||
self.assertEqual(evaluator.grades_for_results(results, judgments), [1, 2])
|
||||
|
||||
|
||||
class TestFixtureValidation(unittest.TestCase):
|
||||
@staticmethod
|
||||
def valid_fixture():
|
||||
case = {
|
||||
"id": "domain-style-minimal",
|
||||
"split": "calibration",
|
||||
"mode": "domain",
|
||||
"domain": "style",
|
||||
"query": "minimal grid",
|
||||
"judgments": [{"identity": {"Style Category": "Minimalism"}, "grade": 2}],
|
||||
}
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"globalNegativeApplicability": {"domains": ["style"], "stacks": []},
|
||||
"cases": [dict(case, id=f"case-{index}") for index in range(60)],
|
||||
}
|
||||
|
||||
def test_valid_schema(self):
|
||||
self.assertEqual(evaluator.validate_fixture(self.valid_fixture(), {"style": {}}, []), [])
|
||||
|
||||
def test_rejects_bad_count_duplicate_id_and_grade(self):
|
||||
fixture = self.valid_fixture()
|
||||
fixture["cases"] = fixture["cases"][:2]
|
||||
fixture["cases"][1]["id"] = fixture["cases"][0]["id"]
|
||||
fixture["cases"][0]["judgments"][0]["grade"] = 3
|
||||
errors = "\n".join(evaluator.validate_fixture(fixture, {"style": {}}, []))
|
||||
self.assertIn("60-100", errors)
|
||||
self.assertIn("duplicate case id", errors)
|
||||
self.assertIn("grade 1 or 2", errors)
|
||||
|
||||
|
||||
class TestThresholdGate(unittest.TestCase):
|
||||
def test_runtime_fingerprint_binds_reasoning_contract(self):
|
||||
original = evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
runtime = root / "src/ui-ux-pro-max/scripts"
|
||||
data = root / "src/ui-ux-pro-max/data"
|
||||
runtime.mkdir(parents=True)
|
||||
data.mkdir(parents=True)
|
||||
for name in ("core.py", "design_system.py", "reasoning_contract.py"):
|
||||
(runtime / name).write_text(name, encoding="utf-8")
|
||||
(data / "styles.csv").write_text("No,Style\n1,Test\n", encoding="utf-8")
|
||||
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = root, runtime, data
|
||||
try:
|
||||
before = evaluator.runtime_fingerprint()
|
||||
(runtime / "reasoning_contract.py").write_text("changed", encoding="utf-8")
|
||||
self.assertNotEqual(before, evaluator.runtime_fingerprint())
|
||||
finally:
|
||||
evaluator.ROOT, evaluator.RUNTIME_DIR, evaluator.DATA_DIR = original
|
||||
|
||||
def test_oracle_fingerprint_hashes_the_selected_cases_file(self):
|
||||
canonical = evaluator.FIXTURE_DIR / "relevance-cases.json"
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
selected = Path(tmp) / "cases.json"
|
||||
selected.write_bytes(canonical.read_bytes())
|
||||
self.assertEqual(
|
||||
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
||||
selected.write_bytes(canonical.read_bytes() + b" ")
|
||||
self.assertNotEqual(
|
||||
evaluator.oracle_fingerprint(selected), evaluator.oracle_fingerprint(canonical))
|
||||
|
||||
def test_metric_sample_and_locked_case_failures_are_actionable(self):
|
||||
report = {
|
||||
"metrics": {"precisionAt1": 0.5},
|
||||
"samples": {"retrieval": 1},
|
||||
"cases": [{"id": "locked", "grades": [0], "actual": [{"Style Category": "Wrong"}]}],
|
||||
}
|
||||
manifest = {
|
||||
"metrics": {"precisionAt1": {"floor": 0.8, "tolerance": 0.01}},
|
||||
"sampleMinimums": {"retrieval": 2},
|
||||
"lockedCases": {"locked": {"withinTop": 1, "minimumGrade": 2}},
|
||||
}
|
||||
manifest["splits"] = {"calibration": {"metrics": {}, "sampleMinimums": {}},
|
||||
"held_out": {"metrics": {}, "sampleMinimums": {}}}
|
||||
report["splits"] = {"calibration": {"metrics": {}, "samples": {}},
|
||||
"held_out": {"metrics": {}, "samples": {}}}
|
||||
failures = evaluator.check_thresholds(report, manifest)
|
||||
self.assertEqual(len(failures), 3)
|
||||
self.assertTrue(any("Wrong" in failure for failure in failures))
|
||||
|
||||
def test_manifest_rejects_missing_contract_sections(self):
|
||||
errors = evaluator.validate_manifest({}, "fingerprint")
|
||||
self.assertTrue(any("missing sections" in error for error in errors))
|
||||
self.assertTrue(any("missing metrics" in error for error in errors))
|
||||
|
||||
def test_manifest_rejects_non_finite_and_invalid_sample_values(self):
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "approved",
|
||||
"approvingMaintainer": "maintainer",
|
||||
"units": "ratios",
|
||||
"splitPolicy": {},
|
||||
"runtimeFingerprint": "fingerprint",
|
||||
"oracleFingerprint": "oracle",
|
||||
"baselineRevision": "97eb2a2",
|
||||
"metrics": {name: {"floor": float("nan")} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": True},
|
||||
"lockedCases": {"case": {}},
|
||||
"splits": {
|
||||
split: {
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
} for split in ("calibration", "held_out")
|
||||
},
|
||||
}
|
||||
errors = evaluator.validate_manifest(manifest, "fingerprint", "oracle")
|
||||
self.assertTrue(any("finite" in error for error in errors))
|
||||
self.assertTrue(any("non-negative integer" in error for error in errors))
|
||||
|
||||
def test_manifest_binds_oracle_and_validates_baseline_revision(self):
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "approved",
|
||||
"approvingMaintainer": "maintainer",
|
||||
"units": "ratios",
|
||||
"splitPolicy": {},
|
||||
"runtimeFingerprint": "runtime",
|
||||
"oracleFingerprint": "wrong",
|
||||
"baselineRevision": "not-a-revision",
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
"lockedCases": {"case": {}},
|
||||
"splits": {
|
||||
split: {
|
||||
"metrics": {name: {"floor": 0.0} for name in evaluator.REQUIRED_METRICS},
|
||||
"sampleMinimums": {"cases": 1},
|
||||
} for split in ("calibration", "held_out")
|
||||
},
|
||||
}
|
||||
errors = evaluator.validate_manifest(manifest, "runtime", "expected")
|
||||
self.assertTrue(any("oracleFingerprint" in error for error in errors))
|
||||
self.assertTrue(any("baselineRevision" in error for error in errors))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression tests for the public style taxonomy and search contract."""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent
|
||||
DATA_DIR = SCRIPTS_DIR.parent / "data"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import search # noqa: E402
|
||||
from design_system import _style_is_dark_primary # noqa: E402
|
||||
|
||||
|
||||
def read_rows(name):
|
||||
with (DATA_DIR / name).open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
class TestStyleTaxonomy(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.styles = read_rows("styles.csv")
|
||||
cls.by_id = {row["Style ID"]: row for row in cls.styles}
|
||||
|
||||
def test_curated_state_distribution_is_explicit(self):
|
||||
counts = {
|
||||
status: sum(row["Status"] == status for row in self.styles)
|
||||
for status in ("active", "supplemental", "deprecated")
|
||||
}
|
||||
self.assertEqual(
|
||||
{"active": 50, "supplemental": 29, "deprecated": 9}, counts
|
||||
)
|
||||
|
||||
def test_every_style_name_and_alias_has_a_deterministic_destination(self):
|
||||
for row in self.styles:
|
||||
queries = [row["Style Category"]]
|
||||
queries.extend(alias for alias in row["Aliases"].split("|") if alias)
|
||||
for query in queries:
|
||||
with self.subTest(style=row["Style ID"], query=query):
|
||||
result = search(query, max_results=1)
|
||||
if (row["Status"] == "deprecated"
|
||||
and row["Replacement Domain"] == "landing"):
|
||||
self.assertEqual(0, result["count"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"domain": row["Replacement Domain"],
|
||||
"id": row["Replacement ID"],
|
||||
},
|
||||
result.get("redirect"),
|
||||
)
|
||||
else:
|
||||
expected_id = (
|
||||
row["Replacement ID"]
|
||||
if row["Status"] == "deprecated"
|
||||
else row["Style ID"]
|
||||
)
|
||||
self.assertEqual(expected_id, result["results"][0]["Style ID"])
|
||||
|
||||
def test_deprecated_rows_never_appear_in_generic_results(self):
|
||||
for query in ("modern interface", "marketing page", "trust design"):
|
||||
with self.subTest(query=query):
|
||||
result = search(query, domain="style", max_results=20)
|
||||
self.assertLessEqual(
|
||||
{row["Status"] for row in result["results"]}, {"active"}
|
||||
)
|
||||
|
||||
def test_style_arbitration_does_not_steal_product_intent(self):
|
||||
result = search("design a financial dashboard for my bank", max_results=1)
|
||||
self.assertEqual("product", result["domain"])
|
||||
|
||||
def test_family_variants_and_mobile_intent_remain_distinct(self):
|
||||
expected_parents = {
|
||||
"gradient-mesh-aurora-evolved": "aurora-ui",
|
||||
"swiss-modernism-2-0": "minimalism-and-swiss-style",
|
||||
"neumorphism-mobile": "neumorphism",
|
||||
"claymorphism-mobile": "claymorphism",
|
||||
"spectrum-2": "spectrum-design-system",
|
||||
}
|
||||
for style_id, parent_id in expected_parents.items():
|
||||
with self.subTest(style=style_id):
|
||||
row = self.by_id[style_id]
|
||||
self.assertEqual(parent_id, row["Parent Style ID"])
|
||||
self.assertIn(row["Status"], {"supplemental", "deprecated"})
|
||||
|
||||
self.assertEqual("style", self.by_id["bento-grids"]["Replacement Domain"])
|
||||
self.assertEqual(
|
||||
"bento-box-grid", self.by_id["bento-grids"]["Replacement ID"]
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
"neumorphism-mobile",
|
||||
search("Neumorphism (Mobile)", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"claymorphism-mobile",
|
||||
search("Claymorphism (Mobile)", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"material-you-md3-mobile",
|
||||
search("M3 Expressive", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"neumorphism-mobile",
|
||||
search("mobile neumorphism UI", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"claymorphism-mobile",
|
||||
search("mobile app with claymorphism", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"spectrum-2",
|
||||
search("design system for Spectrum 2", "style", 1)["results"][0]["Style ID"],
|
||||
)
|
||||
|
||||
def test_claim_fields_use_controlled_non_guarantee_language(self):
|
||||
allowed_performance = {"cost:low", "cost:moderate", "cost:high"}
|
||||
allowed_accessibility = {"risk:low", "risk:conditional", "risk:high"}
|
||||
allowed_mode = {"supported", "conditional", "not-recommended"}
|
||||
for row in self.styles:
|
||||
with self.subTest(style=row["Style ID"]):
|
||||
self.assertIn(row["Performance"].split("|", 1)[0], allowed_performance)
|
||||
self.assertIn(row["Accessibility"].split("|", 1)[0], allowed_accessibility)
|
||||
self.assertIn(row["Light Mode ✓"], allowed_mode)
|
||||
self.assertIn(row["Dark Mode ✓"], allowed_mode)
|
||||
self.assertIn(row["Preferred Mode"], {"auto", "light", "dark"})
|
||||
claim_text = " ".join(row.values())
|
||||
self.assertNotRegex(claim_text, r"(?i)\bWCAG\s+A{2,3}\+?\b")
|
||||
self.assertNotRegex(
|
||||
claim_text,
|
||||
r"(?i)\bWCAG\b.{0,40}\b(?:compliant|compliance)\b",
|
||||
)
|
||||
self.assertNotRegex(row["Framework Compatibility"], r"\d+/10")
|
||||
|
||||
self.assertTrue(_style_is_dark_primary(self.by_id["dark-mode-oled"]))
|
||||
self.assertFalse(
|
||||
_style_is_dark_primary(self.by_id["minimalism-and-swiss-style"])
|
||||
)
|
||||
|
||||
def test_searchable_prompt_lengths_are_balanced(self):
|
||||
lengths_by_type = {}
|
||||
for row in self.styles:
|
||||
length = len(row["AI Prompt Keywords"].split())
|
||||
self.assertLessEqual(length, 40, row["Style ID"])
|
||||
lengths_by_type.setdefault(row["Type"], []).append(length)
|
||||
general_median = statistics.median(lengths_by_type["General"])
|
||||
mobile_median = statistics.median(lengths_by_type["Mobile"])
|
||||
self.assertLessEqual(mobile_median, general_median * 1.6)
|
||||
|
||||
def test_new_rows_have_first_party_provenance(self):
|
||||
payload = json.loads(
|
||||
(DATA_DIR / "data-provenance.json").read_text(encoding="utf-8")
|
||||
)
|
||||
records = {
|
||||
record["entityId"]: record
|
||||
for record in payload["records"]
|
||||
if record["entityKind"] == "style"
|
||||
}
|
||||
new_rows = [row for row in self.styles if int(row["No"]) > 85]
|
||||
self.assertGreaterEqual(len(new_rows), 4)
|
||||
for row in new_rows:
|
||||
with self.subTest(style=row["Style ID"]):
|
||||
record = records[row["Style ID"]]
|
||||
self.assertTrue(record["sources"])
|
||||
self.assertTrue(
|
||||
any(source["type"] == "official" for source in record["sources"])
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Canonical regression contracts for resilient UI text layouts."""
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent
|
||||
DATA_DIR = SCRIPTS_DIR.parent / "data"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import search, search_stack # noqa: E402
|
||||
|
||||
|
||||
def read_rows(relative_path):
|
||||
with (DATA_DIR / relative_path).open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
UX_PHRASES = {
|
||||
"Heading Line Balance": ("progressive visual heuristic", "natural-wrap fallback"),
|
||||
"Long Token Wrapping": ("overflow-wrap anywhere", "text children shrink"),
|
||||
"Text Reflow and Spacing": ("narrow widths zoom", "content-driven height"),
|
||||
"Essential Text Truncation": ("complete access", "visible full-detail path"),
|
||||
"Compact Label Semantics": ("badges communicate state", "meaning and ownership"),
|
||||
"Chip Collection Reflow": ("filter chips", "operable +n disclosure"),
|
||||
"Compact Label Overflow": ("stay whole on one line", "keyboard pointer and touch"),
|
||||
"Compact Control Semantics": ("native role", "pressed or selected state"),
|
||||
"Contextual Live Badge Updates": ("meaningful contextual status", "atomic status"),
|
||||
"Cancellable State Transitions": ("interrupt an in-flight transition", "final semantic state"),
|
||||
}
|
||||
|
||||
TAILWIND_PHRASES = {
|
||||
"Balanced heading wrapping": ("text-balance", "natural wrapping fallback"),
|
||||
"Long token resilience": ("wrap-anywhere", "min-w-0"),
|
||||
"Compact label layout": ("flex flex-wrap gap-2", "whitespace-nowrap", "shrink-0"),
|
||||
}
|
||||
|
||||
|
||||
class TestTextLayoutRetrieval(unittest.TestCase):
|
||||
def test_locked_queries_return_the_canonical_identity_first(self):
|
||||
cases = (
|
||||
("orphan heading line balance", "Heading Line Balance"),
|
||||
("long url token breaks layout", "Long Token Wrapping"),
|
||||
("badge chip label wraps to second line", "Compact Label Overflow"),
|
||||
("filter chip collection reflow hidden values", "Chip Collection Reflow"),
|
||||
("live badge count screen reader", "Contextual Live Badge Updates"),
|
||||
("rapid chip animation interrupted", "Cancellable State Transitions"),
|
||||
)
|
||||
for query, expected in cases:
|
||||
with self.subTest(query=query):
|
||||
result = search(query, domain="ux", max_results=3, diagnostics=True)
|
||||
actual = [row.get("Issue") for row in result["results"]]
|
||||
self.assertTrue(actual, result.get("diagnostics"))
|
||||
self.assertEqual(expected, actual[0], f"ranking={actual!r}")
|
||||
|
||||
def test_tailwind_query_returns_compact_label_layout_first(self):
|
||||
result = search_stack(
|
||||
"chip badge overflow nowrap", "html-tailwind",
|
||||
max_results=3, diagnostics=True,
|
||||
)
|
||||
actual = [row.get("Guideline") for row in result["results"]]
|
||||
self.assertTrue(actual, result.get("diagnostics"))
|
||||
self.assertEqual("Compact label layout", actual[0], f"ranking={actual!r}")
|
||||
|
||||
|
||||
class TestTextLayoutDataContracts(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.ux = read_rows("ux-guidelines.csv")
|
||||
cls.tailwind = read_rows("stacks/html-tailwind.csv")
|
||||
|
||||
def test_new_ux_rows_are_unique_sequential_and_keep_critical_guidance(self):
|
||||
matches = [row for row in self.ux if row["Issue"] in UX_PHRASES]
|
||||
self.assertEqual(len(UX_PHRASES), len(matches))
|
||||
self.assertEqual(len(matches), len({row["Issue"] for row in matches}))
|
||||
self.assertEqual(list(range(110, 120)), [int(row["No"]) for row in matches])
|
||||
for row in matches:
|
||||
with self.subTest(issue=row["Issue"]):
|
||||
text = " ".join(row.values()).casefold()
|
||||
for phrase in UX_PHRASES[row["Issue"]]:
|
||||
self.assertIn(phrase.casefold(), text)
|
||||
self.assertIn(row["Severity"], {"Medium", "High", "Critical"})
|
||||
|
||||
def test_new_tailwind_rows_are_unique_current_and_keep_required_utilities(self):
|
||||
matches = [row for row in self.tailwind if row["Guideline"] in TAILWIND_PHRASES]
|
||||
self.assertEqual(len(TAILWIND_PHRASES), len(matches))
|
||||
self.assertEqual(len(matches), len({row["Guideline"] for row in matches}))
|
||||
self.assertEqual([57, 58, 59], [int(row["No"]) for row in matches])
|
||||
for row in matches:
|
||||
with self.subTest(guideline=row["Guideline"]):
|
||||
text = " ".join(row.values()).casefold()
|
||||
for phrase in TAILWIND_PHRASES[row["Guideline"]]:
|
||||
self.assertIn(phrase.casefold(), text)
|
||||
self.assertEqual("active", row["Status"])
|
||||
self.assertEqual("html-tailwind 4.3", row["Applies To"])
|
||||
self.assertEqual("2026-08-13", row["Verified At"])
|
||||
|
||||
def test_refined_rows_are_context_sensitive_not_universal_recipes(self):
|
||||
expected = {
|
||||
"8": ("depends on distance complexity platform", "shared motion tokens"),
|
||||
"14": ("match how an element changes speed", "linear for constant-rate progress"),
|
||||
"19": ("badges validation text", "stable content-driven container"),
|
||||
"78": ("avoid flashing for near-instant work", "platform and component guidance"),
|
||||
}
|
||||
forbidden = ("use 150-300ms", "operations > 300ms", "linear motion feels robotic")
|
||||
by_number = {row["No"]: row for row in self.ux}
|
||||
for number, phrases in expected.items():
|
||||
with self.subTest(row=number):
|
||||
row = by_number[number]
|
||||
guidance = " ".join((row["Description"], row["Do"])).casefold()
|
||||
for phrase in phrases:
|
||||
self.assertIn(phrase, guidance)
|
||||
for claim in forbidden:
|
||||
self.assertNotIn(claim, guidance)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Freshness and generation-isolation contracts for web stack guidance."""
|
||||
|
||||
import csv
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SCRIPTS_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from core import DATA_DIR, STACK_CONFIG, WEB_STACKS, search_stack # noqa: E402
|
||||
from validate_data import STACK_OFFICIAL_HOSTS # noqa: E402
|
||||
|
||||
CURRENT_APPLICABILITY = {
|
||||
"react": "react 19.2.x",
|
||||
"nextjs": "nextjs 16.2",
|
||||
"vue": "vue 3.5.x",
|
||||
"svelte": "svelte 5",
|
||||
"astro": "astro 7.1.6",
|
||||
"angular": "angular 22.x",
|
||||
"html-tailwind": "html-tailwind 4.3",
|
||||
"shadcn": "shadcn cli 4",
|
||||
"nuxtjs": "nuxtjs 4.5",
|
||||
"nuxt-ui": "nuxt-ui 4.10",
|
||||
}
|
||||
|
||||
|
||||
def _rows(stack):
|
||||
path = DATA_DIR / STACK_CONFIG[stack]["file"]
|
||||
with path.open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle))
|
||||
|
||||
|
||||
class TestWebStackFreshness(unittest.TestCase):
|
||||
def test_web_rows_have_explicit_freshness_metadata(self):
|
||||
for stack in WEB_STACKS:
|
||||
for row in _rows(stack):
|
||||
with self.subTest(stack=stack, row=row["No"]):
|
||||
self.assertIn(row["Status"], {"active", "deprecated"})
|
||||
self.assertTrue(row["Applies To"].startswith(stack))
|
||||
self.assertRegex(row["Verified At"], r"^\d{4}-\d{2}-\d{2}$")
|
||||
self.assertEqual(
|
||||
"legacy" in row["Applies To"].casefold(),
|
||||
row["Status"] == "deprecated",
|
||||
)
|
||||
|
||||
def test_high_impact_rows_use_official_sources(self):
|
||||
for stack in WEB_STACKS:
|
||||
for row in _rows(stack):
|
||||
if row["Severity"] not in {"Critical", "High"}:
|
||||
continue
|
||||
with self.subTest(stack=stack, row=row["No"]):
|
||||
parsed = urlsplit(row["Docs URL"])
|
||||
self.assertEqual(parsed.scheme, "https")
|
||||
self.assertIn(parsed.hostname, STACK_OFFICIAL_HOSTS[stack])
|
||||
|
||||
def test_active_rows_use_the_verified_current_applicability(self):
|
||||
for stack, applicability in CURRENT_APPLICABILITY.items():
|
||||
for row in _rows(stack):
|
||||
if row["Status"] != "active":
|
||||
continue
|
||||
with self.subTest(stack=stack, row=row["No"]):
|
||||
self.assertTrue(row["Applies To"].startswith(applicability))
|
||||
|
||||
def test_svelte_current_and_legacy_queries_do_not_mix_generations(self):
|
||||
current = search_stack("Svelte state props and events", "svelte")
|
||||
legacy = search_stack("Svelte 4 legacy props and events", "svelte")
|
||||
self.assertGreater(current["count"], 0)
|
||||
self.assertGreater(legacy["count"], 0)
|
||||
self.assertEqual({row["Status"] for row in current["results"]}, {"active"})
|
||||
self.assertEqual({row["Status"] for row in legacy["results"]}, {"deprecated"})
|
||||
self.assertTrue(all("legacy" in row["Applies To"] for row in legacy["results"]))
|
||||
|
||||
def test_explicit_old_major_uses_only_curated_legacy_rows(self):
|
||||
cases = {
|
||||
"nextjs": "Next.js 15 middleware auth matcher",
|
||||
"html-tailwind": "Tailwind 3 JIT content configuration",
|
||||
"nuxtjs": "Nuxt 3 app config runtime config migration",
|
||||
}
|
||||
for stack, query in cases.items():
|
||||
with self.subTest(stack=stack):
|
||||
result = search_stack(query, stack)
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual(
|
||||
{row["Status"] for row in result["results"]}, {"deprecated"}
|
||||
)
|
||||
|
||||
def test_current_major_migration_query_stays_on_current_guidance(self):
|
||||
result = search_stack("Next.js 16 migration to proxy", "nextjs")
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual({row["Status"] for row in result["results"]}, {"active"})
|
||||
|
||||
def test_shadcn_named_base_excludes_incompatible_composition_apis(self):
|
||||
result = search_stack("shadcn Base UI asChild composition", "shadcn")
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual(result["results"][0]["Guideline"],
|
||||
"Use render for Base UI composition")
|
||||
self.assertTrue(all("base=radix" not in row["Applies To"]
|
||||
for row in result["results"]))
|
||||
|
||||
def test_common_old_major_syntaxes_select_legacy_guidance(self):
|
||||
cases = {
|
||||
"svelte": "svelte@4 props events",
|
||||
"nextjs": "Next.js (v15) middleware",
|
||||
"html-tailwind": "tailwindcss@3 JIT",
|
||||
"nuxtjs": "nuxt@3 app config",
|
||||
}
|
||||
for stack, query in cases.items():
|
||||
with self.subTest(stack=stack, query=query):
|
||||
result = search_stack(query, stack)
|
||||
self.assertGreater(result["count"], 0)
|
||||
self.assertEqual(
|
||||
{row["Status"] for row in result["results"]}, {"deprecated"}
|
||||
)
|
||||
|
||||
def test_old_major_without_curated_legacy_guidance_abstains(self):
|
||||
result = search_stack("Astro 5 content collections", "astro")
|
||||
self.assertEqual(result["count"], 0)
|
||||
self.assertEqual(result["results"], [])
|
||||
|
||||
def test_current_high_drift_queries_return_current_contracts(self):
|
||||
cases = {
|
||||
("nextjs", "Next.js 16 request interception proxy"): "proxy",
|
||||
("html-tailwind", "Tailwind 4 CSS-first source detection"): "source",
|
||||
("react", "React Effect Event latest values inside an Effect"): "effect event",
|
||||
}
|
||||
for (stack, query), expected in cases.items():
|
||||
with self.subTest(stack=stack):
|
||||
result = search_stack(query, stack, max_results=1)
|
||||
self.assertEqual(result["count"], 1)
|
||||
row = result["results"][0]
|
||||
self.assertEqual(row["Status"], "active")
|
||||
self.assertIn(expected, row["Guideline"].casefold())
|
||||
|
||||
def test_stack_without_curated_legacy_rows_keeps_nonlegacy_fallback(self):
|
||||
result = search_stack(
|
||||
"which Windows UI framework should a new app choose instead of legacy UWP",
|
||||
"uwp",
|
||||
)
|
||||
self.assertGreater(result["count"], 0)
|
||||
|
||||
def test_stale_apis_are_not_recommended_by_active_rows(self):
|
||||
forbidden = {
|
||||
"svelte": ("$: ", "export let ", "on:click", "createeventdispatcher"),
|
||||
"nextjs": ("middleware.ts", "function middleware", "skipmiddleware"),
|
||||
"html-tailwind": (
|
||||
"content: [", "mode: 'jit'", "@tailwindcss/aspect-ratio",
|
||||
"tailwindcss-container-queries",
|
||||
),
|
||||
"nuxt-ui": ("#cell-status", "sortable: true", "v-model:content"),
|
||||
"astro": (
|
||||
"output: 'hybrid'", "viewtransitions", "@astrojs/tailwind",
|
||||
"astro add prefetch",
|
||||
),
|
||||
}
|
||||
for stack, tokens in forbidden.items():
|
||||
for row in _rows(stack):
|
||||
if row["Status"] != "active":
|
||||
continue
|
||||
recommended = " ".join(
|
||||
row[field] for field in ("Guideline", "Do", "Code Good")
|
||||
).casefold()
|
||||
for token in tokens:
|
||||
with self.subTest(stack=stack, row=row["No"], token=token):
|
||||
self.assertNotIn(token, recommended)
|
||||
|
||||
self.assertNotIn(
|
||||
"fetch(url {", _rows("nextjs")[12]["Code Good"].casefold()
|
||||
)
|
||||
self.assertNotIn("catch(e) {}", _rows("react")[39]["Code Good"].casefold())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user