Switch to PyPA simple api for pypi

packages/worker-bundler/src/installer.ts

@@ -11,7 +11,7 @@ import type { FileSystem } from "./file-system";
 import { parse as parseToml } from "smol-toml";
 
 const NPM_REGISTRY = "https://registry.npmjs.org";
-const PYPI_JSON_API = "https://pypi.org/pypi";
+const PYPI_SIMPLE_API = "https://pypi.org/simple";
 const DEFAULT_TIMEOUT_MS = 30000; // 30 seconds
 
 /**
@@ -70,6 +70,20 @@ interface NpmPackageMetadata {
   versions: Record<string, PackageJson>;
 }
 
+interface PypiSimpleFile {
+  filename: string;
+  url: string;
+  hashes?: Record<string, string>;
+  "requires-python"?: string;
+  "core-metadata"?: boolean | { hash?: string; url?: string };
+  yanked?: boolean | string;
+}
+
+interface PypiSimpleMetadata {
+  name: string;
+  files: PypiSimpleFile[];
+}
+
 interface InstallOptions {
   /**
    * Include devDependencies (default: false)
@@ -213,7 +227,7 @@ async function installDependenciesPython(
         fileSystem,
         installedPackages,
         inProgress,
-        PYPI_JSON_API // hardcoding this for now to keep the implementation light
+        PYPI_SIMPLE_API
       )
     )
   );
@@ -353,15 +367,8 @@ async function installPythonPackage(
     try {
       const metadata = await fetchPythonPackageMetadata(name, registry);
 
-      const version = metadata.info.version;
-      const wheel = metadata.urls.find(
-        (url) => url.packagetype === "bdist_wheel"
-      );
-      if (!wheel) {
-        throw new Error(
-          `No wheel distribution found for ${name}@${version} on PyPI`
-        );
-      }
+      const version = metadata.version;
+      const wheel = metadata.wheel;
       const wheelUrl = wheel.url;
 
       const response = await fetchWithTimeout(
@@ -389,6 +396,25 @@ async function installPythonPackage(
       for (const [filePath, content] of Object.entries(packageFilesWheel)) {
         fileSystem.write(`python_modules/${filePath}`, content);
       }
+
+      // Fetch requires_dist from core metadata if available
+      const metadataUrl = getCoreMetadataUrl(wheel);
+      const requiresDist = metadataUrl
+        ? await fetchPythonRequiresDist(metadataUrl)
+        : [];
+
+      await Promise.all(
+        requiresDist.map((dep) =>
+          installPythonPackage(
+            parsePythonVersionString(dep)["name"], // This will change (ie look nicer) after we've completely fleshed out what this should return
+            result,
+            fileSystem,
+            installedPackages,
+            inProgress,
+            PYPI_SIMPLE_API
+          )
+        )
+      );
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       result.warnings.push(`Failed to install ${name}: ${message}`);
@@ -462,10 +488,23 @@ async function fetchPackageMetadata(
   return (await response.json()) as NpmPackageMetadata;
 }
 
-async function fetchPythonPackageMetadata(name: string, registry: string) {
-  // Fetch package metadata from PyPI JSON API
-  // TODO: Redo this to use the PyPA simple repository API
-  const metadataResponse = await fetchWithTimeout(`${registry}/${name}/json`);
+async function fetchPythonPackageMetadata(
+  name: string,
+  registry: string
+): Promise<{ version: string; wheel: PypiSimpleFile }> {
+  // Normalize package name per PEP 503 (lowercase, replace [-_.] with -)
+  const normalizedName = name.toLowerCase().replace(/[-_.]+/g, "-");
+
+  // Fetch package metadata from PyPI Simple API
+  const metadataResponse = await fetchWithTimeout(
+    `${registry}/${normalizedName}/`,
+    {
+      headers: {
+        Accept: "application/vnd.pypi.simple.v1+json"
+      }
+    }
+  );
+
   if (!metadataResponse.ok) {
     const hint =
       metadataResponse.status === 404
@@ -475,16 +514,187 @@ async function fetchPythonPackageMetadata(name: string, registry: string) {
       `PyPI returned ${metadataResponse.status} ${metadataResponse.statusText} for "${name}"${hint}`
     );
   }
-  const metadata = (await metadataResponse.json()) as {
-    info: { version: string };
+  const metadata = (await metadataResponse.json()) as PypiSimpleMetadata;
+
+  const wheel = selectWheel(metadata.files);
+  if (!wheel) {
+    throw new Error(`No compatible wheel found for ${name} on PyPI`);
+  }
+
+  const version = parseWheelVersion(wheel.filename);
+  if (!version) {
+    throw new Error(
+      `Could not parse version from wheel filename: ${wheel.filename}`
+    );
+  }
+
+  return { version, wheel };
+}
+
+/**
+ * Select a compatible wheel from PyPI Simple API files list.
+ * Prefers py3-none-any or py2.py3-none-any wheels for maximum compatibility.
+ * Selects the latest version from compatible wheels.
+ * TODO: implement proper platform/python version matching
+ */
+function selectWheel(files: PypiSimpleFile[]): PypiSimpleFile | undefined {
+  const wheels = files.filter((f) => f.filename.endsWith(".whl"));
+  if (wheels.length === 0) return undefined;
+
+  // Filter to universal wheels (py3-none-any or py2.py3-none-any)
+  const universal = wheels.filter(
+    (w) =>
+      w.filename.includes("-py3-none-any.whl") ||
+      w.filename.includes("-py2.py3-none-any.whl")
+  );
+
+  const candidates = universal.length > 0 ? universal : wheels;
+
+  // Select the wheel with the highest version
+  let latest: PypiSimpleFile | undefined;
+  let latestVersion: string | undefined;
+
+  for (const wheel of candidates) {
+    const version = parseWheelVersion(wheel.filename);
+    if (!version) continue;
+
+    if (
+      !latest ||
+      !latestVersion ||
+      comparePythonVersions(version, latestVersion) > 0
+    ) {
+      latest = wheel;
+      latestVersion = version;
+    }
+  }
+
+  return latest;
+}
 
-    urls: Array<{
-      filename: string;
-      url: string;
-      packagetype: string;
-    }>;
+/**
+ * Compare two PEP 440 version strings.
+ * Returns >0 if a > b, <0 if a < b, 0 if equal.
+ *
+ * Python versions (PEP 440) are not semver-compatible (e.g. "3.6.2.1" has four
+ * release segments), so semver cannot be used here. This is a minimal
+ * comparison: it compares the dotted numeric release segments, and treats
+ * pre-release/dev versions (a/b/rc/alpha/beta/dev/pre) as lower than the same
+ * release so a stable release is preferred.
+ * TODO: full PEP 440 ordering (post-releases, local versions, epochs) if needed.
+ */
+function comparePythonVersions(a: string, b: string): number {
+  const parse = (v: string) => {
+    const releaseMatch = v.match(/^[0-9]+(?:\.[0-9]+)*/);
+    const release = (releaseMatch?.[0] ?? "0").split(".").map(Number);
+    const rest = v.slice(releaseMatch?.[0].length ?? 0);
+    const isPre = /^[.\-_]?(a|b|c|rc|alpha|beta|dev|pre)/i.test(rest);
+    return { release, isPre };
   };
-  return metadata;
+
+  const av = parse(a);
+  const bv = parse(b);
+
+  const len = Math.max(av.release.length, bv.release.length);
+  for (let i = 0; i < len; i++) {
+    const diff = (av.release[i] ?? 0) - (bv.release[i] ?? 0);
+    if (diff !== 0) return diff;
+  }
+
+  // Same release: a stable version outranks a pre-release/dev version
+  if (av.isPre !== bv.isPre) return av.isPre ? -1 : 1;
+  return 0;
+}
+
+/**
+ * Parse version from a wheel filename.
+ * Wheel format: {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
+ *
+ * With no build tag (5 parts): distribution-version-python-abi-platform.whl
+ * With build tag (6+ parts): distribution-version-build-python-abi-platform.whl
+ *
+ * TODO: handle edge cases with distribution names containing hyphens
+ */
+function parseWheelVersion(filename: string): string | undefined {
+  const parts = filename.replace(/\.whl$/, "").split("-");
+  if (parts.length < 5) return undefined;
+
+  // The last three parts are always: python_tag, abi_tag, platform_tag
+  // For 5 parts: distribution, version, py, abi, platform -> version is parts[1]
+  // For 6+ parts: distribution, version, build?, py, abi, platform -> version is parts[1]
+  return parts[1];
+}
+
+/**
+ * Get the core metadata URL from a PyPI Simple API file entry.
+ * Returns undefined if core metadata is not available.
+ *
+ * Per PEP 714: if core-metadata is true or an object (with hash),
+ * metadata is available at {file_url}.metadata unless a separate URL is provided.
+ */
+function getCoreMetadataUrl(file: PypiSimpleFile): string | undefined {
+  const cm = file["core-metadata"];
+  if (!cm) return undefined;
+
+  // If it's an object with an explicit URL, use that
+  if (typeof cm === "object" && cm.url) return cm.url;
+
+  // Otherwise (boolean true or object with just hash), use .metadata suffix
+  if (cm === true || typeof cm === "object") return `${file.url}.metadata`;
+
+  return undefined;
+}
+
+/**
+ * Fetch and parse Requires-Dist from a Python package's core metadata.
+ * Returns empty array if metadata is unavailable or parsing fails.
+ */
+async function fetchPythonRequiresDist(url: string): Promise<string[]> {
+  try {
+    const response = await fetchWithTimeout(url);
+    if (!response.ok) return [];
+    const contentType = response.headers.get("content-type") ?? "";
+    // PyPI tends to send this as `binary/octet-stream` even though it's actually text, this avoids an unhelpful warning
+    const text = contentType.startsWith("text/")
+      ? await response.text()
+      : new TextDecoder().decode(await response.arrayBuffer());
+    return parseRequiresDist(text);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Parse Requires-Dist headers from Python package METADATA file (RFC 822 format).
+ * Handles continuation lines (starting with whitespace).
+ */
+function parseRequiresDist(metadata: string): string[] {
+  const requires: string[] = [];
+  const lines = metadata.split(/\r?\n/);
+  let current: string | undefined;
+
+  for (const raw of lines) {
+    // Continuation line (starts with whitespace)
+    if (raw.startsWith(" ") || raw.startsWith("\t")) {
+      if (current !== undefined) {
+        current += " " + raw.trim();
+      }
+      continue;
+    }
+
+    // Process previous header if it was Requires-Dist
+    if (current !== undefined && current.startsWith("Requires-Dist:")) {
+      requires.push(current.slice("Requires-Dist:".length).trim());
+    }
+
+    current = raw;
+  }
+
+  // Process last header
+  if (current !== undefined && current.startsWith("Requires-Dist:")) {
+    requires.push(current.slice("Requires-Dist:".length).trim());
+  }
+
+  return requires;
 }
 
 /**
@@ -755,6 +965,34 @@ function isTextFile(path: string): boolean {
   return textExtensions.some((ext) => path.toLowerCase().endsWith(ext));
 }
 
+/**
+ * Parse a Python version specifier string (PEP 508) and extract the package name.
+ *
+ * Accepts strings as they appear in `pyproject.toml` `[project].dependencies`
+ * or in PyPI JSON API `info.requires_dist` responses. Examples:
+ *   "requests"
+ *   "requests>=2.0"
+ *   "requests[security]>=2.0"
+ *   "requests (>=2.0)"
+ *   "requests; python_version < '3.8'"
+ *   "requests[security] >= 2.0 ; python_version < '3.8'"
+ *
+ * Returns a tuple of `[package_name, null, null]`. The second and third slots
+ * are placeholders reserved for future use (e.g. extras, version specifier).
+ */
+function parsePythonVersionString(spec: string): { name: string } {
+  // Drop the PEP 508 environment marker (everything after `;`)
+  let head = spec.split(";", 1)[0] ?? "";
+
+  // The package name is the leading run of characters allowed in a PEP 508
+  // identifier: letters, digits, `.`, `-`, `_`. Stop at the first character
+  // that isn't one of those (whitespace, `[`, `(`, `<`, `>`, `=`, `!`, `~`, etc.).
+  const match = head.match(/^\s*([A-Za-z0-9][A-Za-z0-9._-]*)/);
+  const name = match ? match[1]! : head.trim();
+
+  return { name: name };
+}
+
 /**
  * Check if files contain a package.json or pyproject.toml with dependencies that need installing.
  */

packages/worker-bundler/src/tests/e2e.test.ts

@@ -1101,4 +1101,43 @@ describe("createWorker with pyproject.toml", () => {
     expect(body.attrs).toBe("attrs");
     expect(body.attr).toBe("attr");
   });
+
+  it("automatically installs a nested dependency", async () => {
+    const id = "test-worker-" + testId++;
+    const createWorkerResult = await createWorker({
+      files: {
+        "index.py": [
+          "from workers import Response, WorkerEntrypoint",
+          "import typing_inspection",
+          "import typing_extensions",
+          "class Default(WorkerEntrypoint):",
+          "  async def fetch(self, request):",
+          "    return Response.json({",
+          '      "typing_extensions": typing_extensions.__name__,',
+          '      "typing_inspection": typing_inspection.__name__,',
+          "    })"
+        ].join("\n"),
+        "pyproject.toml": [
+          "[project]",
+          'name = "dummy"',
+          'version = "0.0.0"',
+          // typing_inspection depends on typing_extensions
+          'dependencies = ["typing_inspection"]'
+        ].join("\n")
+      }
+    });
+    const worker = env.LOADER.get(id, () => ({
+      mainModule: createWorkerResult.mainModule,
+      modules: createWorkerResult.modules,
+      compatibilityDate: createWorkerResult.wranglerConfig!.compatibilityDate!,
+      compatibilityFlags: createWorkerResult.wranglerConfig!.compatibilityFlags!
+    }));
+    const response = await worker
+      .getEntrypoint()
+      .fetch(new Request("http://worker/"));
+    expect(response.status).toBe(200);
+    const body = (await response.json()) as Record<string, string>;
+    expect(body.typing_extensions).toBe("typing_extensions");
+    expect(body.typing_inspection).toBe("typing_inspection");
+  });
 }, 20000);