Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix typo in CPS's RMSD baseline: was 0.3Å, should have been 0.03Å #225

Merged
merged 2 commits into from
Mar 17, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 15 additions & 32 deletions site/src/lib/RadarChart.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,14 @@
aria-label="Radar chart for adjusting metric weights"
>
<!-- Axes -->
{#each weights as weight, i}
{@const angle = (2 * Math.PI * i) / weights.length}
{#each weights as weight, idx}
{@const angle = (2 * Math.PI * idx) / weights.length}
{@const x = center.x + Math.cos(angle) * radius * 0.8}
{@const y = center.y + Math.sin(angle) * radius * 0.8}
{@const label_radius = idx === 2 ? radius * 1.0 : radius * 0.9}
{@const label_x = center.x + Math.cos(angle) * label_radius}
{@const label_y = center.y + Math.sin(angle) * label_radius}
{@const spacing = idx === 1 ? `1.5em` : `1.2em`}

<line
x1={center.x}
Expand All @@ -302,28 +306,29 @@
/>

<!-- Axis labels -->
{@const label_x = center.x + Math.cos(angle) * radius * 0.9}
{@const label_y = center.y + Math.sin(angle) * radius * 0.9}
<text
x={label_x}
y={label_y}
text-anchor="middle"
dominant-baseline="middle"
font-size="12"
fill="white"
font-size="14"
fill={colors[idx]}
>
<!-- Handle subscripts and superscripts manually since <sub> and <sup> are not supported in SVG -->
{#if weight.label.includes(`<sub>`)}
{@const parts = weight.label.split(/<sub>|<\/sub>/)}
{parts[0]}
<tspan baseline-shift="sub" font-size="8">{parts[1]}</tspan>
<tspan baseline-shift="sub" font-size="10">{parts[1]}</tspan>
{:else if weight.label.includes(`<sup>`)}
{@const parts = weight.label.split(/<sup>|<\/sup>/)}
{parts[0]}
<tspan baseline-shift="super" font-size="8">{parts[1]}</tspan>
<tspan baseline-shift="super" font-size="10">{parts[1]}</tspan>
{:else}
{@html weight.label}
{/if}
<tspan dy={spacing} x={label_x} font-size="12" font-weight="bold"
>{(weight.value * 100).toFixed(0)}%</tspan
>
</text>
{/each}

Expand Down Expand Up @@ -390,40 +395,18 @@
aria-label="Drag to adjust weight balance"
/>
</svg>

<!-- Weight percentages display -->
<div class="weight-display">
{#each weights as weight, idx}
<div class="weight-item" style="color: {colors[idx]}">
<span class="weight-name">{@html weight.label}</span>
<span class="weight-value">{(weight.value * 100).toFixed(0)}%</span>
</div>
{/each}
</div>
</div>

<style>
.radar-chart-container {
display: flex;
flex-direction: column;
align-items: center;
padding: 0.5em;
padding: 0.2em;
margin: 0;
}
svg {
touch-action: none; /* Prevents default touch behaviors */
cursor: pointer; /* Show pointer cursor to hint clickability */
}
.weight-display {
display: flex;
justify-content: space-around;
width: 100%;
margin-top: 0.3em;
}
.weight-item {
display: flex;
flex-direction: column;
place-items: center;
font-size: 0.8em;
font-weight: bold;
}
</style>
31 changes: 12 additions & 19 deletions site/src/lib/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ export const [F1_DEFAULT_WEIGHT, RMSD_DEFAULT_WEIGHT, KAPPA_DEFAULT_WEIGHT] = [

export const DEFAULT_COMBINED_METRIC_CONFIG: CombinedMetricConfig = {
name: `CPS`,
description: `Combined Performance Score weighting discovery, structure optimization, and phonon performance`,
description: `Combined Performance Score weighting discovery (F1), structure optimization (RMSD), and phonon performance (κ<sub>SRME</sub>)`,
weights: [
{
metric: `F1`,
Expand Down Expand Up @@ -131,14 +131,14 @@ function normalize_f1(value: number | undefined): number {
return value // Already in [0,1] range
}

// RMSD is lower=better, with current models in the range of ~0.02-0.25 Å
// RMSD is lower=better, with current models in the range of ~0.01-0.25 Å
// We invert this so that better performance = higher score
function normalize_rmsd(value: number | undefined): number {
if (value === undefined || isNaN(value)) return 0

// Fixed reference points for RMSD (in Å)
const excellent = 0 // Perfect performance (atoms in exact correct positions)
const baseline = 0.3 // in Å, a reasonable baseline for poor performance given worst performing model at time of writing is AlphaNet-MPTrj at 0.0227 Å
const baseline = 0.03 // baseline for poor performance given worst performing model at time of writing is AlphaNet-MPTrj at 0.0227 Å

// Linear interpolation between fixed points with clamping
// Inverse mapping since lower RMSD is better
Expand All @@ -157,26 +157,19 @@ function normalize_kappa_srme(value: number | undefined): number {
return Math.max(0, 1 - value / 2)
}

/**
* Calculate a combined score using normalized metrics weighted by importance factors.
* This uses fixed normalization reference points to ensure score stability when new models are added.
*
* Normalization reference points:
* - F1: Already in [0,1] range, higher is better
* - RMSD: 0.0Å (perfect) to 0.25Å (baseline), lower is better
* - κ_SRME: Range [0,2] linearly mapped to [1,0], lower is better
*
* @param f1 F1 score for discovery
* @param rmsd Root mean square displacement in Å
* @param kappa Symmetric relative mean error for thermal conductivity
* @param config Configuration with weights for each metric
* @returns Combined score between 0-1, or NaN if any weighted metric is missing
*/
// Calculate a combined score using normalized metrics weighted by importance factors.
// This uses fixed normalization reference points to ensure score stability when new models are added.

// Normalization reference points:
// - F1 score for discovery already in [0,1] range, higher is better
// - RMSD Root mean square displacement in range 0Å (perfect) to 0.03Å (baseline), lower is better
// - κ_SRME symmetric relative mean error for lattice thermal conductivity,
// range [0,2] linearly mapped to [1,0], lower is better
export function calculate_combined_score(
f1: number | undefined,
rmsd: number | undefined,
kappa: number | undefined,
config: CombinedMetricConfig,
config: CombinedMetricConfig, // weights for each metric
): number {
// Find weights from config by metric names
const f1_weight =
Expand Down
15 changes: 7 additions & 8 deletions site/src/routes/+page.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -162,8 +162,11 @@
<div class="radar-container">
<div class="radar-header">
<span class="metric-name">{metric_config.name}</span>
<Tooltip text={metric_config.description}>
<Tooltip>
<span class="info-icon">ⓘ</span>
{#snippet tip()}
{@html metric_config.description}
{/snippet}
</Tooltip>

<button
Expand Down Expand Up @@ -269,19 +272,15 @@
max-width: 100%;
background: var(--light-bg);
border-radius: 4px;
padding: 0.2em 0.5em;
padding: 0.1em 0.3em;
box-sizing: border-box;
}

.radar-header {
display: flex;
align-items: center;
gap: 0.5rem;
}

.metric-name {
font-weight: 600;
margin-right: 0.3em;
gap: 6pt;
font-weight: bold;
}

.info-icon {
Expand Down
12 changes: 4 additions & 8 deletions site/tests/landing-page.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,14 +147,10 @@ describe(`Landing Page`, () => {
const axis_lines = document.body.querySelectorAll(`.radar-container svg line`)
expect(axis_lines).toHaveLength(3)

// Check for weight display section
const weight_displays = document.body.querySelectorAll(`.weight-item`)
expect(weight_displays).toHaveLength(3)

// Check the default weight percentages
const weight_values = Array.from(document.body.querySelectorAll(`.weight-value`)).map(
(el) => el.textContent,
)
// Check the default weight percentages in SVG tspan elements
const weight_values = Array.from(
document.body.querySelectorAll(`.radar-container svg tspan:last-child`),
).map((el) => el.textContent)

// Should show the default weights (50%, 40%, 10%)
expect(weight_values).toEqual([`50%`, `40%`, `10%`])
Expand Down
Loading
Loading