From 07a06cd8a3f9acf8414565f3fabbe307bb24a511 Mon Sep 17 00:00:00 2001 From: johannes wasmer Date: Sat, 18 Jul 2026 19:18:14 +0200 Subject: [PATCH] fix: bootstrap MP imports and document setup sequence Make the Materials Project import scripts work from a fresh checkout by updating the pinned MP stack, adding pymatgen compatibility shims, and creating the SQLite schema before import. Also document the required bootstrap/retrain steps and fix the demo results path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- QUICKSTART.md | 23 ++++++++++-- README.md | 24 +++++++++++-- backend/app/core/materials_project.py | 12 +++++-- backend/app/database.py | 7 ++++ backend/app/utils/pymatgen_compat.py | 40 +++++++++++++++++++++ backend/requirements.txt | 5 +-- backend/run_demo.py | 6 ++-- backend/scripts/bulk_fetch_mp.py | 16 ++++++--- backend/scripts/bulk_import_mp.py | 13 +++++-- backend/scripts/fetch_mp_data.py | 52 ++++++++++++++++++++++++++- 10 files changed, 178 insertions(+), 20 deletions(-) create mode 100644 backend/app/utils/pymatgen_compat.py diff --git a/QUICKSTART.md b/QUICKSTART.md index 7b820bb..a0e6211 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -15,7 +15,24 @@ source .venv/bin/activate pip install -r backend/requirements.txt ``` -## Step 2: Verify It Works +## Step 2: Bootstrap the Database & Models + +> Run these once before using the app, demo, or frontend. + +```bash +cd backend + +# Populate the database (requires MP_API_KEY) +python scripts/fetch_mp_data.py --num 100 + +# Optional: full import path +python scripts/bulk_fetch_mp.py --retrain + +# Retrain the .joblib models from the populated DB +python scripts/retrain_models.py +``` + +## Step 3: Verify It Works ```bash # Check database loads (104,842 materials) @@ -43,7 +60,7 @@ Materials: 104842 BaTiO3 band gap = 2.197 eV ``` -## Step 3: Run the Demo (Cathode Discovery) +## Step 4: Run the Demo (Cathode Discovery) ```bash cd atomcraft @@ -62,7 +79,7 @@ This runs the full pipeline: Takes ~5-10 min on a modern laptop. -## Step 4: Try Your Own Exploration +## Step 5: Try Your Own Exploration ### Quick one-liners: diff --git a/README.md b/README.md index b81bdc4..54619ae 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,25 @@ source .venv/bin/activate pip install -r backend/requirements.txt ``` -### Step 2: Verify the Database & Models +### Step 2: Bootstrap the Database & Models + +> The app expects a populated SQLite database and trained model files. +> Run these once before using the demo, API, or frontend. + +```bash +cd backend + +# Populate the Materials Project database (requires MP_API_KEY) +python scripts/fetch_mp_data.py --num 100 + +# Optional: full import path +python scripts/bulk_fetch_mp.py --retrain + +# If you already have data and just want to retrain the .joblib models +python scripts/retrain_models.py +``` + +### Step 3: Verify the Database & Models ```bash cd backend @@ -330,7 +348,7 @@ Materials in DB: 104842 BaTiO3 band gap = 2.197 eV (confidence: 0.850) ``` -### Step 3: Run the Demo +### Step 4: Run the Demo ```bash python backend/run_demo.py @@ -345,7 +363,7 @@ This runs the full cathode discovery pipeline: 6. Generates VASP input files for the best candidate 7. Runs 1 active learning iteration (adds 3 pseudo-validated samples + retrains all 11 models) -### Step 4: Start the Full Stack (Backend + Frontend) +### Step 5: Start the Full Stack (Backend + Frontend) **Option A: Helper script** ```bash diff --git a/backend/app/core/materials_project.py b/backend/app/core/materials_project.py index 9a073f8..1902dac 100644 --- a/backend/app/core/materials_project.py +++ b/backend/app/core/materials_project.py @@ -1,5 +1,6 @@ from typing import Optional from app.config import settings +from app.utils.pymatgen_compat import ensure_pymatgen_compat MP_API_KEY_ENV = "MP_API_KEY" @@ -14,20 +15,25 @@ class MaterialsProjectClient: def __init__(self, api_key: Optional[str] = None): self.api_key = api_key or get_mp_api_key() self._mprester = None + self._init_error = None if self.api_key: try: + ensure_pymatgen_compat() from mp_api.client import MPRester self._mprester = MPRester(api_key=self.api_key) - except ImportError: + except Exception as exc: + self._init_error = str(exc) self._mprester = None def _get_rester(self): if not self._mprester and self.api_key: try: + ensure_pymatgen_compat() from mp_api.client import MPRester self._mprester = MPRester(api_key=self.api_key) - except ImportError: - pass + self._init_error = None + except Exception as exc: + self._init_error = str(exc) return self._mprester async def search_materials( diff --git a/backend/app/database.py b/backend/app/database.py index 361b5c8..5b65119 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -16,3 +16,10 @@ def get_db(): yield db finally: db.close() + + +def init_db() -> None: + """Import models and create all tables for the current database URL.""" + import app.models # noqa: F401 + + Base.metadata.create_all(bind=engine) diff --git a/backend/app/utils/pymatgen_compat.py b/backend/app/utils/pymatgen_compat.py new file mode 100644 index 0000000..fef987c --- /dev/null +++ b/backend/app/utils/pymatgen_compat.py @@ -0,0 +1,40 @@ +"""Compatibility helpers for pymatgen / mp-api / emmet-core imports.""" + +from __future__ import annotations + +import importlib +import sys + + +def _alias_module(alias: str, target: str) -> None: + if alias in sys.modules: + return + sys.modules[alias] = importlib.import_module(target) + + +def ensure_pymatgen_compat() -> None: + """Install module aliases expected by emmet-core on newer pymatgen layouts.""" + aliases = { + "pymatgen.core.graphs": "pymatgen.analysis.graphs", + "pymatgen.core.bond_valence": "pymatgen.analysis.bond_valence", + "pymatgen.core.structure_matcher": "pymatgen.analysis.structure_matcher", + "pymatgen.core.molecule_matcher": "pymatgen.analysis.molecule_matcher", + "pymatgen.core.entries": "pymatgen.entries.computed_entries", + "pymatgen.core.structure_analyzer": "pymatgen.analysis.structure_analyzer", + "pymatgen.core.local_env": "pymatgen.analysis.local_env", + "pymatgen.analysis.compatibility": "pymatgen.entries.compatibility", + } + + for alias, target in aliases.items(): + _alias_module(alias, target) + + emmet_pmg = importlib.import_module("emmet.core.io.pymatgen") + if not hasattr(emmet_pmg, "SymmetryUndeterminedError"): + emmet_pmg.SymmetryUndeterminedError = importlib.import_module( + "pymatgen.symmetry.analyzer" + ).SymmetryUndetermined + + +def patch_mp_api_imports() -> None: + """Public entry point for scripts before importing mp_api.client.""" + ensure_pymatgen_compat() diff --git a/backend/requirements.txt b/backend/requirements.txt index 3ab270f..8462c55 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -2,7 +2,7 @@ fastapi==0.111.0 uvicorn[standard]==0.30.1 sqlalchemy==2.0.31 alembic==1.13.1 -pydantic==2.7.4 +pydantic==2.13.4 pydantic-settings==2.3.4 python-jose[cryptography]==3.3.0 passlib[bcrypt]==1.7.4 @@ -22,7 +22,8 @@ matplotlib==3.9.0 plotly==5.22.0 networkx==3.3 psycopg2-binary==2.9.9 -mp-api==0.41.2 +mp-api==0.46.4 +emmet-core==0.87.1 weasyprint==62.3 pytest==8.2.2 pytest-asyncio==0.23.7 diff --git a/backend/run_demo.py b/backend/run_demo.py index 1523772..08ed6ce 100644 --- a/backend/run_demo.py +++ b/backend/run_demo.py @@ -1,5 +1,6 @@ import sys; sys.path.insert(0, "backend") import asyncio, json, time +from pathlib import Path from app.core.generator import MaterialsGenerator from app.core.predictor import PropertyPredictor @@ -133,6 +134,7 @@ def log(s): # Save results out = "\n".join(results) -with open("backend/demo_results.txt", "w") as f: +results_path = Path(__file__).resolve().parent / "demo_results.txt" +with results_path.open("w") as f: f.write(out) -print(f"\nDemo results saved to backend/demo_results.txt") +print(f"\nDemo results saved to {results_path}") diff --git a/backend/scripts/bulk_fetch_mp.py b/backend/scripts/bulk_fetch_mp.py index 61d9ffc..8d70bbd 100644 --- a/backend/scripts/bulk_fetch_mp.py +++ b/backend/scripts/bulk_fetch_mp.py @@ -7,8 +7,6 @@ import argparse sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from mp_api.client import MPRester - def main(): parser = argparse.ArgumentParser() @@ -20,10 +18,20 @@ def main(): print("ERROR: Set MP_API_KEY environment variable") sys.exit(1) - from app.database import SessionLocal + from app.utils.pymatgen_compat import ensure_pymatgen_compat + ensure_pymatgen_compat() + from mp_api.client import MPRester + from app.database import SessionLocal, init_db from app.models.material import Material, Property - client = MPRester(api_key=api_key) + try: + client = MPRester(api_key=api_key) + except Exception as exc: + print("ERROR: Materials Project client initialization failed.") + print(f"Client error: {exc}") + return + + init_db() total = client.materials.summary.count() num_chunks = (total // 1000) + 1 print(f"Total in MP: {total} materials ({num_chunks} chunks of 1000)") diff --git a/backend/scripts/bulk_import_mp.py b/backend/scripts/bulk_import_mp.py index 06ab456..d0049d2 100644 --- a/backend/scripts/bulk_import_mp.py +++ b/backend/scripts/bulk_import_mp.py @@ -15,12 +15,21 @@ def main(): print("ERROR: Set MP_API_KEY") sys.exit(1) + from app.utils.pymatgen_compat import ensure_pymatgen_compat + ensure_pymatgen_compat() from mp_api.client import MPRester - from app.database import SessionLocal + from app.database import SessionLocal, init_db from app.models.material import Material, Property from sqlalchemy import insert - client = MPRester(api_key=api_key) + try: + client = MPRester(api_key=api_key) + except Exception as exc: + print("ERROR: Materials Project client initialization failed.") + print(f"Client error: {exc}") + return + + init_db() total = client.materials.summary.count() num_chunks = (total // 1000) + 1 print(f"MP: {total} materials ({num_chunks} chunks of 1000)") diff --git a/backend/scripts/fetch_mp_data.py b/backend/scripts/fetch_mp_data.py index c69277b..3410ec8 100644 --- a/backend/scripts/fetch_mp_data.py +++ b/backend/scripts/fetch_mp_data.py @@ -16,9 +16,55 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from app.core.materials_project import mp_client -from app.database import SessionLocal +from app.database import SessionLocal, init_db from app.models.material import Material, Composition, Property from app.core.trainer import train_and_save_models +from app.utils.pymatgen_compat import ensure_pymatgen_compat + + +def _check_materials_project_client() -> bool: + init_error = getattr(mp_client, "_init_error", None) + if init_error: + print("ERROR: Materials Project client initialization failed.") + print(f"Client error: {init_error}") + print("") + print( + "This usually means the MP API key is invalid or the installed " + "mp-api/emmet-core/pymatgen versions are incompatible." + ) + return False + + try: + ensure_pymatgen_compat() + from mp_api.client import MPRester + except ModuleNotFoundError as exc: + print("ERROR: Materials Project client could not be imported.") + print(f"Import error: {exc}") + print("") + print( + "This usually means the installed mp-api/emmet-core/pymatgen " + "versions are incompatible." + ) + print( + "In this checkout, the pinned pymatgen version is not enough by itself " + "to guarantee that mp_api.client imports cleanly." + ) + print("") + print("Try:") + print(" 1. Recreate the venv from backend/requirements.txt") + print(" 2. Check `python -c \"from mp_api.client import MPRester\"`") + print(" 3. If that fails, update the mp-api/emmet-core/pymatgen pins together") + return False + + try: + if mp_client.api_key: + MPRester(api_key=mp_client.api_key) + except Exception as exc: + print("ERROR: Materials Project client initialization failed.") + print(f"Import succeeded, but MPRester could not be created: {exc}") + return False + + return True async def fetch_and_store(num_materials: int = 100, retrain: bool = False): @@ -28,6 +74,10 @@ async def fetch_and_store(num_materials: int = 100, retrain: bool = False): print("Then: export MP_API_KEY='your_key'") return + if not _check_materials_project_client(): + return + + init_db() print(f"Fetching up to {num_materials} materials from Materials Project...") data = []