fix: bootstrap MP imports and document setup sequence

QUICKSTART.md

@@ -15,7 +15,24 @@ source .venv/bin/activate
 pip install -r backend/requirements.txt
 ```
 
-## Step 2: Verify It Works
+## Step 2: Bootstrap the Database & Models
+
+> Run these once before using the app, demo, or frontend.
+
+```bash
+cd backend
+
+# Populate the database (requires MP_API_KEY)
+python scripts/fetch_mp_data.py --num 100
+
+# Optional: full import path
+python scripts/bulk_fetch_mp.py --retrain
+
+# Retrain the .joblib models from the populated DB
+python scripts/retrain_models.py
+```
+
+## Step 3: Verify It Works
 
 ```bash
 # Check database loads (104,842 materials)
@@ -43,7 +60,7 @@ Materials: 104842
 BaTiO3 band gap = 2.197 eV
 ```
 
-## Step 3: Run the Demo (Cathode Discovery)
+## Step 4: Run the Demo (Cathode Discovery)
 
 ```bash
 cd atomcraft
@@ -62,7 +79,7 @@ This runs the full pipeline:
 
 Takes ~5-10 min on a modern laptop.
 
-## Step 4: Try Your Own Exploration
+## Step 5: Try Your Own Exploration
 
 ### Quick one-liners:
 

README.md

@@ -301,7 +301,25 @@ source .venv/bin/activate
 pip install -r backend/requirements.txt
 ```
 
-### Step 2: Verify the Database & Models
+### Step 2: Bootstrap the Database & Models
+
+> The app expects a populated SQLite database and trained model files.
+> Run these once before using the demo, API, or frontend.
+
+```bash
+cd backend
+
+# Populate the Materials Project database (requires MP_API_KEY)
+python scripts/fetch_mp_data.py --num 100
+
+# Optional: full import path
+python scripts/bulk_fetch_mp.py --retrain
+
+# If you already have data and just want to retrain the .joblib models
+python scripts/retrain_models.py
+```
+
+### Step 3: Verify the Database & Models
 
 ```bash
 cd backend
@@ -330,7 +348,7 @@ Materials in DB: 104842
 BaTiO3 band gap = 2.197 eV (confidence: 0.850)
 ```
 
-### Step 3: Run the Demo
+### Step 4: Run the Demo
 
 ```bash
 python backend/run_demo.py
@@ -345,7 +363,7 @@ This runs the full cathode discovery pipeline:
 6. Generates VASP input files for the best candidate
 7. Runs 1 active learning iteration (adds 3 pseudo-validated samples + retrains all 11 models)
 
-### Step 4: Start the Full Stack (Backend + Frontend)
+### Step 5: Start the Full Stack (Backend + Frontend)
 
 **Option A: Helper script**
 ```bash

backend/app/core/materials_project.py

@@ -1,5 +1,6 @@
 from typing import Optional
 from app.config import settings
+from app.utils.pymatgen_compat import ensure_pymatgen_compat
 
 
 MP_API_KEY_ENV = "MP_API_KEY"
@@ -14,20 +15,25 @@ class MaterialsProjectClient:
     def __init__(self, api_key: Optional[str] = None):
         self.api_key = api_key or get_mp_api_key()
         self._mprester = None
+        self._init_error = None
         if self.api_key:
             try:
+                ensure_pymatgen_compat()
                 from mp_api.client import MPRester
                 self._mprester = MPRester(api_key=self.api_key)
-            except ImportError:
+            except Exception as exc:
+                self._init_error = str(exc)
                 self._mprester = None
 
     def _get_rester(self):
         if not self._mprester and self.api_key:
             try:
+                ensure_pymatgen_compat()
                 from mp_api.client import MPRester
                 self._mprester = MPRester(api_key=self.api_key)
-            except ImportError:
-                pass
+                self._init_error = None
+            except Exception as exc:
+                self._init_error = str(exc)
         return self._mprester
 
     async def search_materials(

backend/app/database.py

@@ -16,3 +16,10 @@ def get_db():
         yield db
     finally:
         db.close()
+
+
+def init_db() -> None:
+    """Import models and create all tables for the current database URL."""
+    import app.models  # noqa: F401
+
+    Base.metadata.create_all(bind=engine)

backend/app/utils/pymatgen_compat.py

@@ -0,0 +1,40 @@
+"""Compatibility helpers for pymatgen / mp-api / emmet-core imports."""
+
+from __future__ import annotations
+
+import importlib
+import sys
+
+
+def _alias_module(alias: str, target: str) -> None:
+    if alias in sys.modules:
+        return
+    sys.modules[alias] = importlib.import_module(target)
+
+
+def ensure_pymatgen_compat() -> None:
+    """Install module aliases expected by emmet-core on newer pymatgen layouts."""
+    aliases = {
+        "pymatgen.core.graphs": "pymatgen.analysis.graphs",
+        "pymatgen.core.bond_valence": "pymatgen.analysis.bond_valence",
+        "pymatgen.core.structure_matcher": "pymatgen.analysis.structure_matcher",
+        "pymatgen.core.molecule_matcher": "pymatgen.analysis.molecule_matcher",
+        "pymatgen.core.entries": "pymatgen.entries.computed_entries",
+        "pymatgen.core.structure_analyzer": "pymatgen.analysis.structure_analyzer",
+        "pymatgen.core.local_env": "pymatgen.analysis.local_env",
+        "pymatgen.analysis.compatibility": "pymatgen.entries.compatibility",
+    }
+
+    for alias, target in aliases.items():
+        _alias_module(alias, target)
+
+    emmet_pmg = importlib.import_module("emmet.core.io.pymatgen")
+    if not hasattr(emmet_pmg, "SymmetryUndeterminedError"):
+        emmet_pmg.SymmetryUndeterminedError = importlib.import_module(
+            "pymatgen.symmetry.analyzer"
+        ).SymmetryUndetermined
+
+
+def patch_mp_api_imports() -> None:
+    """Public entry point for scripts before importing mp_api.client."""
+    ensure_pymatgen_compat()

backend/requirements.txt

@@ -2,7 +2,7 @@ fastapi==0.111.0
 uvicorn[standard]==0.30.1
 sqlalchemy==2.0.31
 alembic==1.13.1
-pydantic==2.7.4
+pydantic==2.13.4
 pydantic-settings==2.3.4
 python-jose[cryptography]==3.3.0
 passlib[bcrypt]==1.7.4
@@ -22,7 +22,8 @@ matplotlib==3.9.0
 plotly==5.22.0
 networkx==3.3
 psycopg2-binary==2.9.9
-mp-api==0.41.2
+mp-api==0.46.4
+emmet-core==0.87.1
 weasyprint==62.3
 pytest==8.2.2
 pytest-asyncio==0.23.7

backend/run_demo.py

@@ -1,5 +1,6 @@
 import sys; sys.path.insert(0, "backend")
 import asyncio, json, time
+from pathlib import Path
 
 from app.core.generator import MaterialsGenerator
 from app.core.predictor import PropertyPredictor
@@ -133,6 +134,7 @@ def log(s):
 
 # Save results
 out = "\n".join(results)
-with open("backend/demo_results.txt", "w") as f:
+results_path = Path(__file__).resolve().parent / "demo_results.txt"
+with results_path.open("w") as f:
     f.write(out)
-print(f"\nDemo results saved to backend/demo_results.txt")
+print(f"\nDemo results saved to {results_path}")

backend/scripts/bulk_fetch_mp.py

@@ -7,8 +7,6 @@
 import argparse
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
-from mp_api.client import MPRester
-
 
 def main():
     parser = argparse.ArgumentParser()
@@ -20,10 +18,20 @@ def main():
         print("ERROR: Set MP_API_KEY environment variable")
         sys.exit(1)
 
-    from app.database import SessionLocal
+    from app.utils.pymatgen_compat import ensure_pymatgen_compat
+    ensure_pymatgen_compat()
+    from mp_api.client import MPRester
+    from app.database import SessionLocal, init_db
     from app.models.material import Material, Property
 
-    client = MPRester(api_key=api_key)
+    try:
+        client = MPRester(api_key=api_key)
+    except Exception as exc:
+        print("ERROR: Materials Project client initialization failed.")
+        print(f"Client error: {exc}")
+        return
+
+    init_db()
     total = client.materials.summary.count()
     num_chunks = (total // 1000) + 1
     print(f"Total in MP: {total} materials ({num_chunks} chunks of 1000)")

backend/scripts/bulk_import_mp.py

@@ -15,12 +15,21 @@ def main():
         print("ERROR: Set MP_API_KEY")
         sys.exit(1)
 
+    from app.utils.pymatgen_compat import ensure_pymatgen_compat
+    ensure_pymatgen_compat()
     from mp_api.client import MPRester
-    from app.database import SessionLocal
+    from app.database import SessionLocal, init_db
     from app.models.material import Material, Property
     from sqlalchemy import insert
 
-    client = MPRester(api_key=api_key)
+    try:
+        client = MPRester(api_key=api_key)
+    except Exception as exc:
+        print("ERROR: Materials Project client initialization failed.")
+        print(f"Client error: {exc}")
+        return
+
+    init_db()
     total = client.materials.summary.count()
     num_chunks = (total // 1000) + 1
     print(f"MP: {total} materials ({num_chunks} chunks of 1000)")

backend/scripts/fetch_mp_data.py

@@ -16,9 +16,55 @@
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from app.core.materials_project import mp_client
-from app.database import SessionLocal
+from app.database import SessionLocal, init_db
 from app.models.material import Material, Composition, Property
 from app.core.trainer import train_and_save_models
+from app.utils.pymatgen_compat import ensure_pymatgen_compat
+
+
+def _check_materials_project_client() -> bool:
+    init_error = getattr(mp_client, "_init_error", None)
+    if init_error:
+        print("ERROR: Materials Project client initialization failed.")
+        print(f"Client error: {init_error}")
+        print("")
+        print(
+            "This usually means the MP API key is invalid or the installed "
+            "mp-api/emmet-core/pymatgen versions are incompatible."
+        )
+        return False
+
+    try:
+        ensure_pymatgen_compat()
+        from mp_api.client import MPRester
+    except ModuleNotFoundError as exc:
+        print("ERROR: Materials Project client could not be imported.")
+        print(f"Import error: {exc}")
+        print("")
+        print(
+            "This usually means the installed mp-api/emmet-core/pymatgen "
+            "versions are incompatible."
+        )
+        print(
+            "In this checkout, the pinned pymatgen version is not enough by itself "
+            "to guarantee that mp_api.client imports cleanly."
+        )
+        print("")
+        print("Try:")
+        print("  1. Recreate the venv from backend/requirements.txt")
+        print("  2. Check `python -c \"from mp_api.client import MPRester\"`")
+        print("  3. If that fails, update the mp-api/emmet-core/pymatgen pins together")
+        return False
+
+    try:
+        if mp_client.api_key:
+            MPRester(api_key=mp_client.api_key)
+    except Exception as exc:
+        print("ERROR: Materials Project client initialization failed.")
+        print(f"Import succeeded, but MPRester could not be created: {exc}")
+        return False
+
+    return True
 
 
 async def fetch_and_store(num_materials: int = 100, retrain: bool = False):
@@ -28,6 +74,10 @@ async def fetch_and_store(num_materials: int = 100, retrain: bool = False):
         print("Then: export MP_API_KEY='your_key'")
         return
 
+    if not _check_materials_project_client():
+        return
+
+    init_db()
     print(f"Fetching up to {num_materials} materials from Materials Project...")
 
     data = []