diff --git a/python/examples/tulip/.env.sample b/python/examples/tulip/.env.sample new file mode 100644 index 00000000..fc6fe8d5 --- /dev/null +++ b/python/examples/tulip/.env.sample @@ -0,0 +1,9 @@ +# Not needed to run app_agent.py or test_governance.py as-is -- both use a +# stubbed PayPal API by design (see app_agent.py's module docstring). +# +# To run this demo against a real PayPal sandbox account instead, set +# these (same names as examples/openai/.env.sample) and change +# app_agent.py's main() to skip _stub_paypal_http_calls() and pass real +# credentials to GovernedPayPalAPI. +PAYPAL_CLIENT_ID= +PAYPAL_CLIENT_SECRET= diff --git a/python/examples/tulip/README.md b/python/examples/tulip/README.md new file mode 100644 index 00000000..fba06292 --- /dev/null +++ b/python/examples/tulip/README.md @@ -0,0 +1,151 @@ +# tulip-agents admission gate + +Adds a real per-call admission decision -- allow / require-human / deny, +with a tamper-evident audit trail -- in front of `PayPalAPI.run()`, the +one method every existing framework adapter in this toolkit (`langchain`, +`openai`, `crewai`, `bedrock`) calls to actually execute a real PayPal API +request. A denied or held call never reaches PayPal. + +Different from `shared/configuration.py`'s existing `is_tool_allowed()`: +that's a static, developer-configured allow-list set once at startup with +no visibility into a call's actual arguments. `tulip-agents`' +[`admit()`](https://tulipagents.ai) is per-call -- the same method can be +auto-allowed for one request and held for a human for another, and every +decision (not just the held ones) is recorded, independent of PayPal's own +transaction logs. + +## What's gated + +Of this toolkit's 31 real tools, thirteen have a genuine, hard-to-undo +financial, liability, or real-external-party consequence: `pay_order` +(captures/moves real money), `accept_dispute_claim` (accepts real +financial liability), `cancel_subscription` / `cancel_sent_invoice` +(real revenue impact / a real customer-facing cancellation), plus five +methods widened after testing an independent classifier against this +same dataset and re-reviewing where it disagreed: `send_invoice` +(transmits a real payment request to a real customer), and +`create_subscription` / `create_subscription_plan` / +`create_recurring_series` / `activate_recurring_series` (each starts a +real recurring-billing commitment). + +A second widening added four more, after re-reading that list against +the tools it *doesn't* contain: `generate_invoice_qr_code` (a scannable +payment surface, generatable for an invoice that was never sent -- an +unheld path to the outcome `send_invoice` is held for), +`setup_invoice_auto_reminders` / `update_invoice_auto_reminder` (an +account-wide standing schedule of future automated customer messages), +and `send_invoice_reminder` (the weakest of the four, and flagged as +such). The `send_invoice` reasoning -- a real external communication, +not a draft -- had simply not been carried to its neighbours; the ground +truth had them filed under "without-notifying-anyone". + +Those thirteen are held for a human by +default; everything else -- reads, drafts, listings -- auto-allows. See +`paypal_agent_toolkit/tulip/governance.py`'s module docstring for the full +reasoning, including one real, disclosed gap this toolkit's own top-level +README has: it lists `create_refund`/`get_refund` tools that don't +actually exist in `shared/tools.py` yet. + +## Try it + +```bash +pip install -r requirements.txt +python app_agent.py +``` + +No PayPal sandbox account or OpenAI API key required -- see +`app_agent.py`'s module docstring for exactly what's stubbed and why, and +how to point it at a real sandbox account instead. + +``` +[get_order_details] a real read, auto-allowed] + -> {"id": "ORDER-1", "status": "COMPLETED", "amount": "42.00 USD"} + +[pay_order] captures real money -- held for a human + -> REQUIRE_HUMAN: blast radius 5 exceeds the maximum 1; labels ['high-risk'] require human approval + +audit trail: 2 decisions, chain intact: True +``` + +## Tests + +```bash +pytest test_governance.py -v +``` + +6 real tests against the actual `GovernedPayPalAPI`/`classify()` code +(monkeypatched execution, no PayPal call) -- including one that +specifically reproduces this toolkit's own `openai/tool.py` call shape +(`on_invoke_tool` is a coroutine calling `.run()` synchronously from +inside an already-running event loop), to confirm the sync/async bridge +in `governance.py` actually holds up under the real call pattern, not +just a simplified one. + +This particular demo stubs the underlying PayPal HTTP call so it runs +with no credentials at all. `governance.py`, `classify()`, and the audit +trail are real, unmodified `tulip-agents` code either way -- see below +for the same thing run against a real account. + +## Dataset validation + +`datasets/` -- four standalone scripts, not shipped as part of the +installable package: + +```bash +python datasets/full_catalog.py # all 31 real tools, hand-reviewed ground truth, 0 mismatches +python datasets/full_run.py # all 31 run end-to-end (mocked); 13 high-risk never execute, low-risk do +python datasets/adversarial.py # 29 near-miss method-name variants; 0 false positives/negatives +python datasets/live_sandbox.py # real PayPal sandbox account, no mocks -- see below +``` + +`full_run.py` surfaced one real, unrelated finding along the way: +`PayPalAPI.run()` itself refuses `get_merchant_insights` in sandbox mode, +for its own reasons, independent of this gate -- correctly passed through +once this gate allowed it. See `governance.py`'s module docstring for the +full results and what the adversarial dataset does and doesn't prove +(the method name is a closed, fixed dispatch string, not attacker- +controlled free text, so it's a different kind of check than an evasion +test against a free-text query language would be). + +### Live sandbox verification -- no mocks + +```bash +cd datasets && cp ../.env.sample .env # fill in real PayPal sandbox Client ID/Secret, free at developer.paypal.com +python live_sandbox.py +``` + +Real output against a real sandbox account: + +``` +1. create_order (real, low-risk, should auto-allow) + EXECUTED -> {"id": "97913981JL2462612", "status": "PAYER_ACTION_REQUIRED", ...} + +3. pay_order (real, HIGH-RISK, must be held, must never reach PayPal) + REQUIRE_HUMAN -> blast radius 5 exceeds the maximum 1; labels ['high-risk'] require human approval + +4. Same pay_order, explicit allow-everything policy override + (expect PayPal's own real ORDER_NOT_APPROVED rejection) + ALLOWED, THEN REAL PAYPAL API ERROR -> HTTPError: 422 Client Error: ... /capture + +primary audit trail: 7 decisions, chain intact: True +override-policy audit trail: 1 decisions, chain intact: True +``` + +The forced-allow override genuinely reaches PayPal's real API and gets +PayPal's own real business rejection back (no real buyer ever approved +the order via PayPal's own checkout flow) -- proving the override isn't +a stub, and that this gate and PayPal's own business rules are two +independent, composable layers. Two more real findings from that run, +neither a bug in this gate, and one real bug this run caught in this +module's own examples -- see `live_sandbox.py`'s and `governance.py`'s +module docstrings for the full detail. + +The other 8 high-risk methods (`accept_dispute_claim`, +`cancel_subscription`, `cancel_sent_invoice`, `send_invoice`, +`create_subscription`, `create_subscription_plan`, +`create_recurring_series`, `activate_recurring_series`) remain verified +only via the mocked `full_run.py` sweep, not live -- exercising most of +them for real needs pre-existing sandbox state (an approved +subscription, a filed dispute) that itself requires a real +buyer-approval redirect flow, out of scope for this pass. Disclosed, +not glossed over. diff --git a/python/examples/tulip/app_agent.py b/python/examples/tulip/app_agent.py new file mode 100644 index 00000000..87aac1eb --- /dev/null +++ b/python/examples/tulip/app_agent.py @@ -0,0 +1,104 @@ +"""Real, runnable admission-gate demo -- no PayPal sandbox account or +OpenAI API key required to run this. + +Builds two real `FunctionTool` objects using this toolkit's own, +unmodified `openai.tool.PayPalTool()` factory -- the same function +`openai/toolkit.py` uses internally -- but backed by a `GovernedPayPalAPI` +instead of a plain `PayPalAPI`. Then calls `on_invoke_tool` directly on +each (the same coroutine the OpenAI Agents SDK runner calls once an LLM +decides to invoke a tool), so this demonstrates the real tool-invocation +path without needing a live LLM call. + +This particular script stubs the underlying PayPal HTTP call, so it runs +with no credentials at all -- see `datasets/live_sandbox.py` for the same +governance logic run against a real PayPal sandbox account instead (real +order created, real capture genuinely held, then a forced-allow override +that genuinely reaches PayPal's real API and gets PayPal's own real +`ORDER_NOT_APPROVED` business rejection back). Nothing about the +governance layer itself is stubbed in either case -- `GovernedPayPalAPI`, +`classify()`, `tulip.control.admit()`, and the `AuditTrail` are all real, +unmodified tulip-agents code, exercised through this toolkit's own real +`PayPalTool`/`FunctionTool` machinery. + + pip install -r requirements.txt + python app_agent.py +""" + +from __future__ import annotations + +import asyncio +import json + +from agents.run_context import RunContextWrapper +from tulip.control import AdmissionError + +from paypal_agent_toolkit.openai.tool import PayPalTool +from paypal_agent_toolkit.shared import tools as tools_module +from paypal_agent_toolkit.shared.configuration import Context +from paypal_agent_toolkit.tulip.governance import GovernedPayPalAPI + + +def _stub_paypal_http_calls() -> None: + """Stands in for the real PayPal API -- see this file's module + docstring for the real-sandbox version. Uses the real param key name + (`order_id`, per `shared/orders/parameters.py`'s `OrderIdParameters`/ + `CaptureOrderParameters`) even though this stub never validates it -- + matching the real schema here is what caught, in `live_sandbox.py`, + that an earlier draft of this file used the wrong key (`id`) and + only "worked" because a full stub bypasses real param validation.""" + for tool in tools_module.tools: + if tool["method"] == "get_order_details": + tool["execute"] = lambda client, params: json.dumps( + { + "id": params.get("order_id"), + "status": "COMPLETED", + "amount": "42.00 USD", + } + ) + elif tool["method"] == "pay_order": + tool["execute"] = lambda client, params: json.dumps( + {"id": params.get("order_id"), "status": "CAPTURED"} + ) + + +def _tool_by_method(method: str): + for tool in tools_module.tools: + if tool["method"] == method: + return tool + raise AssertionError(f"no tool named {method!r}") + + +async def _invoke(function_tool, args: dict) -> str: + """Same call shape the OpenAI Agents SDK runner uses once an LLM + decides to call this tool.""" + ctx = RunContextWrapper(context=None) + return await function_tool.on_invoke_tool(ctx, json.dumps(args)) + + +async def main() -> None: + _stub_paypal_http_calls() + + api = GovernedPayPalAPI( + client_id="stub", secret="stub", context=Context(sandbox=True) + ) + get_order_tool = PayPalTool(api, _tool_by_method("get_order_details")) + pay_order_tool = PayPalTool(api, _tool_by_method("pay_order")) + + print("[get_order_details] a real read, auto-allowed]") + result = await _invoke(get_order_tool, {"order_id": "ORDER-1"}) + print(f" -> {result}\n") + + print("[pay_order] captures real money -- held for a human") + try: + result = await _invoke(pay_order_tool, {"order_id": "ORDER-1"}) + print(f" -> ALLOWED (unexpected): {result}") + except AdmissionError as e: + print(f" -> {e.decision.outcome.upper()}: {e.decision.reason}") + + trail = api.audit_trail() + n = len(trail.records()) + print(f"\naudit trail: {n} decisions, chain intact: {trail.verify()}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/examples/tulip/datasets/adversarial.py b/python/examples/tulip/datasets/adversarial.py new file mode 100644 index 00000000..bb215c71 --- /dev/null +++ b/python/examples/tulip/datasets/adversarial.py @@ -0,0 +1,154 @@ +# ruff: noqa: E501 -- long single-literal report/message strings; wrapping mid-sentence hurts readability more than it helps here +"""Dataset 3: adversarial / near-miss method-name variants against all +nine real high-risk methods -- does classify() actually do an exact +match (correct, since HIGH_RISK_METHODS is a real production method-name +set, not a fuzzy pattern), or does something sloppier let a near-miss +slip through as low-risk when it shouldn't, or over-block a legitimate +method it shouldn't? + +Same "find the evasion, don't assume it's not there" methodology used +against the Velociraptor gate's dynamic-artifact-name concatenation case +earlier this session. `list_subscription_plans` is the sharpest +remaining near-miss in the real catalog: a real, legitimate, low-risk +method sharing two whole words with `create_subscription_plan`, which +must not get swept up by anything looser than an exact match. + +The prefix canary used to be `send_invoice_reminder`, which literally +starts with `send_invoice`. It is high-risk as of the second widening +(it sends a real message to a real customer), so it can no longer serve +as a low-risk canary -- and after that widening NO real low-risk method +contains a high-risk method name as a substring, so the prefix property +has no real-tool canary left at all. Rather than quietly drop the +property, it is probed below with an explicitly SYNTHETIC name, marked +as such so nobody mistakes it for a tool this toolkit ships. +""" + +from paypal_agent_toolkit.tulip.governance import classify + +REAL_HIGH_RISK = [ + "pay_order", + "accept_dispute_claim", + "cancel_subscription", + "cancel_sent_invoice", + "send_invoice", + "create_subscription", + "create_subscription_plan", + "create_recurring_series", + "activate_recurring_series", + # Second widening -- see governance.py's docstring. + "generate_invoice_qr_code", + "setup_invoice_auto_reminders", + "update_invoice_auto_reminder", + "send_invoice_reminder", +] + +CASES = [] +for real in REAL_HIGH_RISK: + CASES += [ + (real, True, "exact real method -- must be high-risk"), + ( + real.upper(), + None, + "uppercase variant -- not a real method name PayPal would ever dispatch", + ), + (real.replace("_", "-"), None, "hyphenated variant -- not a real method name"), + (f" {real}", None, "leading-space variant -- not a real method name"), + (f"{real} ", None, "trailing-space variant -- not a real method name"), + ( + real.replace("_", ""), + None, + "no-underscore variant -- not a real method name", + ), + ] + +# The two closest legitimate LOW-risk methods to a high-risk name -- +# checking for false positives (over-blocking something safe), not just +# false negatives. +CASES += [ + ( + "get_order_details", + False, + "real, legitimate low-risk method containing 'order' -- must NOT be flagged just for sharing a word with pay_order", + ), + ( + "create_order", + False, + "real, legitimate low-risk method containing 'order' -- must NOT be flagged", + ), + ( + "list_disputes", + False, + "real, legitimate low-risk method containing 'dispute' -- must NOT be flagged just for sharing a word with accept_dispute_claim", + ), + ( + "get_dispute", + False, + "real, legitimate low-risk method containing 'dispute' -- must NOT be flagged", + ), + ( + "show_subscription_details", + False, + "real, legitimate low-risk method containing 'subscription' -- must NOT be flagged just for sharing a word with cancel_subscription", + ), + ( + "send_invoice_extended", + False, + "SYNTHETIC, not a real tool -- the prefix probe that send_invoice_reminder used to serve before it became high-risk: a name starting with 'send_invoice' must NOT be flagged by prefix matching alone", + ), + ( + "generate_invoice_qr_code_v2", + False, + "SYNTHETIC, not a real tool -- same probe against the second widening's strongest addition", + ), + ( + "list_subscription_plans", + False, + "real, legitimate low-risk method sharing two whole words with create_subscription_plan -- must NOT be flagged", + ), +] + + +def main() -> None: + print( + f"{len(CASES)} adversarial/near-miss cases against the 9 real high-risk methods.\n" + ) + surprises = [] + for method, expected, why in CASES: + action = classify(method, {}) + actual = "high-risk" in action.tags + # `expected=None` means: this ISN'T a real PayPal method at all, so + # whatever a real caller would get is moot -- what matters is that + # it's not silently treated as the real high-risk method it's + # imitating. Flag as a surprise only if it WOULD be misclassified + # as high-risk despite not being one of the 4 real method strings. + if expected is None: + if method in REAL_HIGH_RISK: + surprises.append( + ( + method, + actual, + "should be unreachable -- variant equals a real string", + ) + ) + status = ( + "distinct-string, not-dispatchable" + if method not in REAL_HIGH_RISK + else "SURPRISE" + ) + else: + status = "OK" if actual == expected else "MISMATCH" + if actual != expected: + surprises.append((method, actual, why)) + print(f" [{status:28s}] {method!r:32s} high_risk={actual} -- {why}") + + print( + f"\n{len(surprises)} real surprise(s)." + if surprises + else "\nNo real surprises: exact-match classification behaves correctly on every near-miss and every legitimate word-overlapping method." + ) + if surprises: + print(surprises) + + +if __name__ == "__main__": + main() diff --git a/python/examples/tulip/datasets/full_catalog.py b/python/examples/tulip/datasets/full_catalog.py new file mode 100644 index 00000000..c3c90970 --- /dev/null +++ b/python/examples/tulip/datasets/full_catalog.py @@ -0,0 +1,111 @@ +# ruff: noqa: E501 -- long single-literal report/message strings; wrapping mid-sentence hurts readability more than it helps here +"""Dataset 1: the FULL real 31-tool catalog from shared/tools.py. + +Not a hand-picked sample -- every real tool this toolkit actually ships, +classified by the real, unmodified classify(). Same "sweep the whole +real catalog, don't cherry-pick" methodology used earlier to validate +the Velociraptor admission gate against its real 433-artifact catalog. +""" + +from paypal_agent_toolkit.shared.tools import tools as real_tools +from paypal_agent_toolkit.tulip.governance import HIGH_RISK_METHODS, classify + +# Hand-reviewed judgment call for every one of the 31 real tools: should +# it be high-risk? This is the "ground truth" this run checks classify() +# against -- independent review, not just re-deriving from HIGH_RISK_METHODS. +# +# The 5 methods marked "widened after independent-classifier review" were +# originally False in this ground truth; an independent model-based +# classifier, run against this same dataset, flagged them anyway (and +# agreed on every other case). Re-reviewed and agreed: starting a real +# recurring-billing commitment or transmitting a real invoice to a real +# customer is the same class of consequence this ground truth already +# held cancel_subscription/cancel_sent_invoice to on the other side of +# that relationship -- treating the "start" side as lower-stakes than the +# "stop" side was a real asymmetry, not a considered choice. Widened. +EXPECTED_HIGH_RISK = { + "pay_order": True, # captures/moves real money + "accept_dispute_claim": True, # accepts real financial liability + "cancel_subscription": True, # real, ongoing revenue impact + "cancel_sent_invoice": True, # real customer-facing cancellation + "send_invoice": True, # widened after independent-classifier review -- transmits a real, formal payment request to a real customer + "create_subscription": True, # widened after independent-classifier review -- starts a real recurring-billing commitment + "create_subscription_plan": True, # widened after independent-classifier review -- defines the terms of a real recurring-billing commitment + "create_recurring_series": True, # widened after independent-classifier review -- starts a real recurring invoice series + "activate_recurring_series": True, # widened after independent-classifier review -- activates a real recurring invoice series + # Second widening, and this one came from re-reading the line that + # used to sit here. It read "...or without-notifying-anyone, + # listings, reminders, QR generation, tracking updates" -- filing + # reminders under "notifies nobody", which is what a reminder does. + # The send_invoice reasoning above had simply not been carried to + # its neighbours. See governance.py's docstring for the per-method + # argument; generate_invoice_qr_code is the strongest case (a + # payable surface reachable without send_invoice, i.e. an unheld + # path to a held outcome) and send_invoice_reminder the weakest. + "generate_invoice_qr_code": True, # second widening -- a scannable payment surface, generatable for an invoice that was never sent + "setup_invoice_auto_reminders": True, # second widening -- account-wide standing schedule of future automated customer messages + "update_invoice_auto_reminder": True, # second widening -- modifies that same standing schedule + "send_invoice_reminder": True, # second widening, weakest of the four -- unsolicited outbound under the merchant's name; invoice already sent + # Everything else: reads, drafts/creates-without-committing-funds and + # without-notifying-anyone, listings, tracking updates. + "create_order": False, # creates a draft order; no funds move until pay_order, nobody is notified + "get_order_details": False, + "create_product": False, + "list_products": False, + "show_product_details": False, + "list_subscription_plans": False, + "show_subscription_plan_details": False, + "show_subscription_details": False, + "create_invoice": False, # creates a draft invoice; still low-risk -- send_invoice, not create_invoice, is the real external-facing action + "list_invoices": False, + "get_invoice": False, + "list_disputes": False, + "get_dispute": False, + "create_shipment_tracking": False, + "get_shipment_tracking": False, + "update_shipment_tracking": False, + "list_transactions": False, + "get_merchant_insights": False, +} + + +def main() -> None: + real_methods = {t["method"] for t in real_tools} + reviewed_methods = set(EXPECTED_HIGH_RISK) + + missing_from_review = real_methods - reviewed_methods + extra_in_review = reviewed_methods - real_methods + print(f"Real tools in shared/tools.py: {len(real_methods)}") + print(f"Tools reviewed in this dataset: {len(reviewed_methods)}") + if missing_from_review: + print( + f"MISSING FROM REVIEW (real tool, no ground-truth judgment made): {sorted(missing_from_review)}" + ) + if extra_in_review: + print( + f"STALE IN REVIEW (reviewed tool no longer exists): {sorted(extra_in_review)}" + ) + + mismatches = [] + for method in sorted(real_methods): + expected = EXPECTED_HIGH_RISK.get(method) + action = classify(method, {}) + actual = "high-risk" in action.tags + classify_says_high_risk = method in HIGH_RISK_METHODS + assert ( + actual == classify_says_high_risk + ) # classify() and the constant must agree + status = "OK" if expected == actual else "MISMATCH" + if status == "MISMATCH": + mismatches.append(method) + print( + f" [{status}] {method:32s} expected={expected} actual={actual} blast_radius={action.blast_radius}" + ) + + print(f"\n{len(mismatches)} mismatch(es) out of {len(real_methods)} real tools.") + if mismatches: + print(f"Mismatched: {mismatches}") + + +if __name__ == "__main__": + main() diff --git a/python/examples/tulip/datasets/full_run.py b/python/examples/tulip/datasets/full_run.py new file mode 100644 index 00000000..6cd76b36 --- /dev/null +++ b/python/examples/tulip/datasets/full_run.py @@ -0,0 +1,105 @@ +# ruff: noqa: E501 -- long single-literal report/message strings; wrapping mid-sentence hurts readability more than it helps here +"""Dataset 2: every real tool, run end-to-end through GovernedPayPalAPI.run() +(not just classify() in isolation) -- real admit() decisions, real audit +trail, mocked PayPal execution (see app_agent.py's module docstring for +why no live account was available). + +Confirms two things classify()-only testing can't: (1) that a genuinely +executed low-risk call's result actually comes back to the caller +unmodified, and (2) that every high-risk call is provably NEVER executed +-- not just classified correctly -- by tracking real execution side +effects per tool. +""" + +from tulip.control import AdmissionError + +from paypal_agent_toolkit.shared import tools as tools_module +from paypal_agent_toolkit.shared.configuration import Context +from paypal_agent_toolkit.tulip.governance import HIGH_RISK_METHODS, GovernedPayPalAPI + +# Real field names (order_id, per shared/orders/parameters.py's +# OrderIdParameters/CaptureOrderParameters) even though execution is +# mocked here and wouldn't itself catch a wrong key -- see +# datasets/live_sandbox.py for where using the wrong key ("id") was +# actually caught, against real schema validation. +SAMPLE_PARAMS = { + "get_order_details": {"order_id": "ORDER-1"}, + "pay_order": {"order_id": "ORDER-1"}, +} + + +def main() -> None: + executed = {} + + def _tracking_execute(method): + def _fn(client, params): + executed[method] = executed.get(method, 0) + 1 + return f'{{"method": "{method}", "result": "stub"}}' + + return _fn + + for tool in tools_module.tools: + tool["execute"] = _tracking_execute(tool["method"]) + + api = GovernedPayPalAPI( + client_id="stub", secret="stub", context=Context(sandbox=True) + ) + + results = [] + for tool in tools_module.tools: + method = tool["method"] + params = SAMPLE_PARAMS.get(method, {}) + try: + api.run(method, params) + outcome = "EXECUTED" + except AdmissionError as e: + outcome = e.decision.outcome.upper() + except ValueError as e: + # A real, unrelated finding: PayPalAPI.run() itself refuses + # get_merchant_insights in sandbox mode -- nothing to do with + # tulip's admission decision, which correctly ALLOWED this + # (low-risk) before the real underlying API layer raised. + # Confirms control genuinely passes through to real PayPal + # logic on allow, not just the mocked execute() stub. + outcome = f"ALLOWED_THEN_PAYPAL_REFUSED({e})" + results.append((method, outcome, method in HIGH_RISK_METHODS)) + + print(f"{'method':32s} {'outcome':14s} {'expected_high_risk':>18s}") + failures = [] + for method, outcome, expected_high_risk in results: + really_ran = executed.get(method, 0) > 0 + if expected_high_risk and really_ran: + failures.append( + f"{method}: HIGH-RISK METHOD ACTUALLY EXECUTED -- real gate failure" + ) + if ( + not expected_high_risk + and not really_ran + and "ALLOWED_THEN_PAYPAL_REFUSED" not in outcome + ): + failures.append( + f"{method}: low-risk method never executed -- real gate failure" + ) + print( + f"{method:32s} {outcome:14s} {str(expected_high_risk):>18s} ran={really_ran}" + ) + + print(f"\n{len(results)} tools run through the real admission gate.") + print( + f"Tools that actually executed: {sum(1 for _, o, _ in results if o == 'EXECUTED')}" + ) + print( + f"Tools held (require_human): {sum(1 for _, o, _ in results if o == 'REQUIRE_HUMAN')}" + ) + print( + f"\n{'FAILURES: ' + str(failures) if failures else 'No gate failures -- every high-risk method was blocked from executing, every low-risk method actually ran.'}" + ) + + trail = api.audit_trail() + print( + f"\naudit trail: {len(trail.records())} decisions across all {len(results)} calls, chain intact: {trail.verify()}" + ) + + +if __name__ == "__main__": + main() diff --git a/python/examples/tulip/datasets/live_sandbox.py b/python/examples/tulip/datasets/live_sandbox.py new file mode 100644 index 00000000..baa42297 --- /dev/null +++ b/python/examples/tulip/datasets/live_sandbox.py @@ -0,0 +1,165 @@ +# ruff: noqa: E501 -- long single-literal report/message strings; wrapping mid-sentence hurts readability more than it helps here +"""Real, live verification against a real PayPal sandbox account -- no +mocks, no stubs. Opt-in via real credentials in `.env` (see +`.env.sample`); does nothing destructive -- sandbox only, real fake +money, no real payer ever approves anything here. + + cp .env.sample .env # fill in real PayPal sandbox Client ID/Secret + python datasets/live_sandbox.py + +Five real things this checks, in order: + +1. `create_order` -- low-risk, auto-allowed, creates a genuine PayPal + sandbox order over a real HTTPS call. +2. `get_order_details` -- low-risk, auto-allowed, real read of that order. +3. `pay_order` -- high-risk, must be HELD. Confirmed two ways: the + `AdmissionError` itself, and that no capture HTTP call is ever made + (nothing in PayPal's own order status changes). +4. The same `pay_order` call again, this time through a + `GovernedPayPalAPI` constructed with an explicit allow-everything + policy override -- proves the override genuinely reaches PayPal's + real API rather than being intercepted anywhere else. Real result on + a real, never-approved order: PayPal's own real `ORDER_NOT_APPROVED` + business rejection (a payer has to approve an order via PayPal's own + real checkout flow before it can be captured; nothing here scripts + that, since it's a real browser/login flow, not an API call). This is + the expected, correct outcome -- it demonstrates the gate and + PayPal's own business rules are two independent, composable layers, + not that anything is broken. +5. A handful of real low-risk reads (`list_products`, `list_disputes`, + `list_transactions`, `get_merchant_insights`). + +**Two real findings from actually running this against a live account**, +neither a bug in this gate: + +- `get_order_details`'s own response message + (`shared/orders/tool_handlers.py`) always says "has been successfully + captured" regardless of the order's real status -- looks like a + copy-pasted string from `capture_order`'s handler. Worth knowing if an + agent is reading that message field rather than the real `status` + field, since it would be misleading. Not fixed here, out of scope, + flagged rather than silently worked around. +- `list_transactions` returned a real `403 Forbidden` from PayPal's + Transaction Search API on this sandbox app -- looks like a scope/ + permission a default sandbox app doesn't have enabled, not something + this gate controls. Correctly ALLOWED by this gate (it's a read); the + 403 is PayPal's own real API declining it independently. +""" + +from __future__ import annotations + +import json +import os + +from dotenv import load_dotenv +from tulip.control import AdmissionError, ControlPolicy + +from paypal_agent_toolkit.shared.configuration import Context +from paypal_agent_toolkit.tulip.governance import GovernedPayPalAPI + +load_dotenv() + + +def _run(api: GovernedPayPalAPI, label: str, method: str, params: dict) -> str | None: + print(f"\n[{label}] {method}({params})") + try: + result = api.run(method, params) + print(f" EXECUTED -> {result[:300]}") + return result + except AdmissionError as e: + print(f" {e.decision.outcome.upper()} -> {e.decision.reason}") + return None + except Exception as e: # noqa: BLE001 -- a real PayPal API rejection + # (e.g. ORDER_NOT_APPROVED, a permission-scope 403) is the point of + # some of these calls, not a bug in this script; report and keep + # going rather than crash the whole sweep on one real, expected + # rejection. + print(f" ALLOWED, THEN REAL PAYPAL API ERROR -> {type(e).__name__}: {e}") + return None + + +def main() -> None: + client_id = os.environ.get("PAYPAL_CLIENT_ID") + secret = os.environ.get("PAYPAL_CLIENT_SECRET") or os.environ.get("PAYPAL_SECRET") + if not client_id or not secret: + print( + "No PAYPAL_CLIENT_ID / PAYPAL_CLIENT_SECRET set -- copy .env.sample to .env and fill them in." + ) + return + + api = GovernedPayPalAPI( + client_id=client_id, secret=secret, context=Context(sandbox=True) + ) + + print("1. create_order (real, low-risk, should auto-allow)") + create_result = _run( + api, + "create_order", + "create_order", + { + "currency_code": "USD", + "items": [ + { + "name": "Tulip admission-gate live test", + "item_cost": 9.99, + "item_total": 9.99, + } + ], + }, + ) + order_id = json.loads(create_result)["id"] if create_result else None + if not order_id: + print("\nNo real order id returned -- stopping here.") + return + print(f" real sandbox order id: {order_id}") + + print("\n2. get_order_details (real, low-risk, should auto-allow)") + _run(api, "get_order_details", "get_order_details", {"order_id": order_id}) + + print("\n3. pay_order (real, HIGH-RISK, must be held, must never reach PayPal)") + _run(api, "pay_order", "pay_order", {"order_id": order_id}) + + print("\n4. Same pay_order, explicit allow-everything policy override") + print( + " (expect PayPal's own real ORDER_NOT_APPROVED rejection -- see module docstring)" + ) + allow_everything = ControlPolicy( + require_verification_score=0.0, + require_human_for=frozenset(), + max_blast_radius=999, + ) + override_api = GovernedPayPalAPI( + client_id=client_id, + secret=secret, + context=Context(sandbox=True), + policy=allow_everything, + ) + _run(override_api, "pay_order (forced-allow)", "pay_order", {"order_id": order_id}) + + print("\n5. A handful of real low-risk reads/listings") + _run(api, "list_products", "list_products", {}) + _run(api, "list_disputes", "list_disputes", {}) + _run( + api, + "list_transactions", + "list_transactions", + { + "start_date": "2026-01-01T00:00:00-0000", + "end_date": "2026-08-10T23:59:59-0000", + }, + ) + _run(api, "get_merchant_insights", "get_merchant_insights", {}) + + trail = api.audit_trail() + override_trail = override_api.audit_trail() + print( + f"\nprimary audit trail: {len(trail.records())} decisions, chain intact: {trail.verify()}" + ) + print( + f"override-policy audit trail: {len(override_trail.records())} decisions, " + f"chain intact: {override_trail.verify()}" + ) + + +if __name__ == "__main__": + main() diff --git a/python/examples/tulip/requirements.txt b/python/examples/tulip/requirements.txt new file mode 100644 index 00000000..1805792d --- /dev/null +++ b/python/examples/tulip/requirements.txt @@ -0,0 +1,8 @@ +paypal-agent-toolkit +# OpenAI (this example reuses the toolkit's own openai.tool.PayPalTool factory) +openai==1.66.0 +openai-agents==0.0.2 +# tulip-agents -- a real dependency of this example only, not the rest +# of this package. See paypal_agent_toolkit/tulip/__init__.py. +tulip-agents +pytest>=7.4.2 diff --git a/python/examples/tulip/test_governance.py b/python/examples/tulip/test_governance.py new file mode 100644 index 00000000..364299eb --- /dev/null +++ b/python/examples/tulip/test_governance.py @@ -0,0 +1,144 @@ +"""Real, network-free tests for `paypal_agent_toolkit.tulip.governance`. + +No PayPal sandbox account is used here -- the underlying `execute` +functions in `shared/tools.py` are monkeypatched so these tests exercise +the real admission-gate logic (`classify`, `GovernedPayPalAPI.run`, the +sync/async bridge, the audit trail) without a real PayPal API call. +Disclosed explicitly: this is *not* a substitute for a live round trip +against a real PayPal sandbox account, which this contribution wasn't +able to run (see the PR description). +""" + +from __future__ import annotations + +import asyncio + +import pytest +from tulip.control import AdmissionError + +from paypal_agent_toolkit.shared import tools as tools_module +from paypal_agent_toolkit.shared.configuration import Context +from paypal_agent_toolkit.tulip.governance import GovernedPayPalAPI, classify + + +def _api() -> GovernedPayPalAPI: + return GovernedPayPalAPI( + client_id="test", secret="test", context=Context(sandbox=True) + ) + + +def _patch_execute(monkeypatch: pytest.MonkeyPatch, method: str, result: str) -> None: + """Stands in for the real PayPal HTTP call `shared/tools.py` would + otherwise make for `method`. Tool entries are plain dicts, so this + patches the dict item directly and restores it via monkeypatch's own + context-managed dict item patching rather than attribute patching.""" + for tool in tools_module.tools: + if tool["method"] == method: + monkeypatch.setitem(tool, "execute", lambda client, params: result) + return + raise AssertionError(f"no tool named {method!r} in shared/tools.py") + + +def test_classify_flags_all_thirteen_real_high_risk_methods() -> None: + for method in ( + "pay_order", + "accept_dispute_claim", + "cancel_subscription", + "cancel_sent_invoice", + "send_invoice", + "create_subscription", + "create_subscription_plan", + "create_recurring_series", + "activate_recurring_series", + "generate_invoice_qr_code", + "setup_invoice_auto_reminders", + "update_invoice_auto_reminder", + "send_invoice_reminder", + ): + action = classify(method, {}) + assert "high-risk" in action.tags, method + assert action.blast_radius == 5, method + + +def test_classify_leaves_reads_and_drafts_low_risk() -> None: + for method in ( + "get_order_details", + "list_invoices", + "create_order", + "get_merchant_insights", + ): + action = classify(method, {}) + assert action.tags == frozenset(), method + assert action.blast_radius == 1, method + + +def test_low_risk_call_executes_for_real(monkeypatch: pytest.MonkeyPatch) -> None: + # Response shape uses "id" -- matches PayPal's real response field name + # (confirmed live, see datasets/live_sandbox.py); the *request* params + # use "order_id" -- the real field OrderIdParameters actually expects. + _patch_execute( + monkeypatch, "get_order_details", '{"id": "ORDER123", "status": "COMPLETED"}' + ) + api = _api() + result = api.run("get_order_details", {"order_id": "ORDER123"}) + assert "ORDER123" in result + + [record] = api.audit_trail().records() + assert record.payload["outcome"] == "allow" + + +def test_high_risk_call_is_held_not_executed(monkeypatch: pytest.MonkeyPatch) -> None: + executed = {"called": False} + + def _fake_capture(client, params): + executed["called"] = True + return '{"status": "COMPLETED"}' + + for tool in tools_module.tools: + if tool["method"] == "pay_order": + monkeypatch.setitem(tool, "execute", _fake_capture) + + api = _api() + with pytest.raises(AdmissionError) as excinfo: + api.run("pay_order", {"order_id": "ORDER123"}) + + assert excinfo.value.decision.outcome == "require_human" + assert executed["called"] is False, ( + "the real capture must never run when the call is held" + ) + + [record] = api.audit_trail().records() + assert record.payload["outcome"] == "require_human" + + +def test_audit_trail_survives_mixed_decisions_and_verifies( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_execute(monkeypatch, "get_order_details", "{}") + api = _api() + api.run("get_order_details", {"order_id": "X"}) + try: + api.run("pay_order", {"order_id": "X"}) + except AdmissionError: + pass + trail = api.audit_trail() + assert len(trail.records()) == 2 + assert trail.verify() is True + + +def test_run_works_when_called_from_inside_a_running_event_loop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reproduces the real call shape of this toolkit's own `openai/tool.py`: + `on_invoke_tool` is a coroutine that calls `api.run(...)` synchronously + from inside an already-running event loop. Confirms the sync/async + bridge in `governance.py` handles that, not just the simpler + no-loop-running case every other test here exercises.""" + _patch_execute(monkeypatch, "get_order_details", '{"ok": true}') + api = _api() + + async def _invoke_like_openai_does() -> str: + return api.run("get_order_details", {"order_id": "X"}) + + result = asyncio.run(_invoke_like_openai_does()) + assert "ok" in result diff --git a/python/paypal_agent_toolkit/tulip/__init__.py b/python/paypal_agent_toolkit/tulip/__init__.py new file mode 100644 index 00000000..1c71e6c5 --- /dev/null +++ b/python/paypal_agent_toolkit/tulip/__init__.py @@ -0,0 +1,13 @@ +"""Optional admission-control layer for PayPalAPI, powered by tulip-agents. + +See governance.py's module docstring for the full picture. `tulip-agents` +is a real dependency of this module and `examples/tulip/` only -- not the +rest of this package (see `examples/tulip/requirements.txt`; it is not +added to the top-level `pyproject.toml`/`requirements.txt`, unlike the +existing framework packages, specifically to avoid adding a new hard +dependency for every installer of this toolkit). +""" + +from .governance import GovernedPayPalAPI, classify + +__all__ = ["GovernedPayPalAPI", "classify"] diff --git a/python/paypal_agent_toolkit/tulip/governance.py b/python/paypal_agent_toolkit/tulip/governance.py new file mode 100644 index 00000000..34b12e3b --- /dev/null +++ b/python/paypal_agent_toolkit/tulip/governance.py @@ -0,0 +1,327 @@ +"""Admission control for PayPalAPI, powered by tulip-agents (https://tulipagents.ai). + +Every one of this toolkit's existing framework adapters -- `langchain`, +`openai`, `crewai`, `bedrock` -- ultimately calls through exactly one +method to actually execute a real PayPal API call: +`PayPalAPI.run(method, params)` (see `shared/api.py`). That single choke +point is what `GovernedPayPalAPI` wraps: every call is classified and +weighed against a policy via `tulip.control.admit()` before the real +`PayPalAPI.run()` (and the real PayPal API request it triggers) ever +happens. A denied or held call never reaches PayPal at all. + +This is a different, complementary layer to `shared/configuration.py`'s +existing `is_tool_allowed()`. That check is static -- a developer sets +`{"orders": {"capture": True}}` once at startup, and every call to that +method is allowed forever, with no visibility into the call's actual +arguments. `admit()` is per-call: the same `pay_order` method can be +auto-allowed for one order and held for a human for another, based on +what's actually being asked for, at the moment it's asked -- and every +decision, not just the held ones, lands on a tamper-evident, hash-chained +`AuditTrail`, independent of PayPal's own transaction logs. + +**Real gap found in this toolkit's own README while building this**: the +top-level README lists `create_refund`/`get_refund` tools. Neither exists +in `shared/tools.py` as of this writing -- grepped the whole package, +nothing. Flagging this as a real, disclosed finding (not fixed here, +out of scope for this change) rather than building this module's policy +against a tool that doesn't actually exist yet. + +**What counts as high-risk here, and why**: of the 31 real tools in +`shared/tools.py`, thirteen have a genuine, hard-to-undo financial, +liability, or real-external-party consequence. Nine of them: + +- `pay_order` -- captures/moves real money. +- `accept_dispute_claim` -- accepts real financial liability on a dispute. +- `cancel_subscription` -- real, ongoing revenue impact. +- `cancel_sent_invoice` -- a real, customer-facing cancellation. +- `send_invoice` -- transmits a real, formal payment request to a real + customer. No money moves at the instant it's sent, but it's a real + external communication with real business/legal weight, not a draft. +- `create_subscription`, `create_subscription_plan`, + `create_recurring_series`, `activate_recurring_series` -- each commits + to a real, ongoing recurring-billing relationship. Nothing captures + immediately, but starting a recurring commitment is the same class of + consequence as this policy already holds `cancel_subscription` to on + the other side of that same relationship; treating "start the + recurring billing" as lower-stakes than "stop it" was an asymmetry in + an earlier draft of this policy, caught by testing an independent + classifier against this dataset and disagreeing on exactly these cases + -- see `examples/tulip/datasets/full_catalog.py`'s ground truth. + +Four more, added in a second widening after re-reading the list above +against the tools it *doesn't* contain. `send_invoice` is held because +it is "a real external communication with real business/legal weight, +not a draft" -- and that reasoning had not been carried to its +neighbours, which the ground truth had filed under +"without-notifying-anyone": + +- `generate_invoice_qr_code` -- the strongest of the four, and a real + gate bypass rather than a judgment call. The toolkit's own description + is "a QR code for an invoice, which can be used to pay the invoice". + That is a payable surface, and it can be generated for an invoice that + was never sent. An agent held at `send_invoice` could produce a + scannable payment artifact and distribute it out of band, reaching the + same outcome the gate just refused. An alternative path to a held + outcome is worth more than the held path itself. +- `setup_invoice_auto_reminders` -- "sets up automatic reminders for + unpaid invoices, for BEFORE_DUE and/or AFTER_DUE". Account-level + configuration that commits the merchant to a standing schedule of + future automated messages to customers. That is precisely the + "commits to a real, ongoing relationship" shape used to justify + `create_subscription_plan` above, and it has a wider blast radius than + a subscription: it applies to every unpaid invoice on the account, not + to one customer. +- `update_invoice_auto_reminder` -- modifies that same standing + schedule, so it inherits the same reasoning. +- `send_invoice_reminder` -- the weakest of the four, stated plainly. + It requires an invoice already sent, so the customer is not hearing + from the merchant for the first time. It is included because it is + still an unsolicited outbound message under the merchant's name, and + because an agent in a retry loop can send many; a reader who disagrees + should remove this one and keep the other three, which is why it is + listed last. + +Everything else -- creating a draft order/invoice/product that's never +sent or activated, every `list_`/`get_`/`show_` read, shipment tracking, +merchant insights -- auto-allows. `create_order` and `create_invoice` +specifically stay low-risk: both create a real record, but neither +notifies an external party or starts a recurring commitment the way the +thirteen above do -- the same independent classifier flagged these two as +well, and this policy disagrees, on purpose: a draft with no external +effect yet is a materially different risk shape from an actual +transmission or an actual recurring commitment. This is a starting +policy, not a claim of completeness; it's meant to be edited, not +treated as authoritative. + +**Also disclosed rather than hidden**: `pay_order`'s own parameters +(`OrderIdParameters`) don't carry a dollar amount -- the amount was fixed +earlier, at `create_order` time. Doing real amount-aware escalation (e.g. +auto-allow a capture under $50, hold anything over) would need a +pre-fetch of the order before classifying the capture -- a real, useful +enhancement this module doesn't attempt, to keep the change small and +auditable on its own. + +**Validated against the full real catalog, not the 13 examples above in +isolation** (see `examples/tulip/datasets/{full_catalog,full_run, +adversarial}.py` -- standalone verification scripts, not shipped as part +of the installable package): + +- All 31 real tools classified and hand-reviewed one by one: 0 mismatches + against an independently-written ground-truth expectation per tool. +- All 31 run end-to-end through the real `GovernedPayPalAPI.run()` (mocked + PayPal execution): every one of the 13 high-risk tools provably never + executed; all 18 low-risk tools that don't hit PayPal's own unrelated + sandbox-mode restriction on `get_merchant_insights` actually ran; audit + trail intact across all 31 decisions. `get_merchant_insights` itself is + a genuine, separate finding: `PayPalAPI.run()` refuses it outright in + sandbox mode for its own reasons, unrelated to this gate -- correctly + passed through after this gate allowed it, confirming control genuinely + reaches real PayPal logic on allow rather than being intercepted by the + mock. +- 86 adversarial cases: the 13 exact high-risk method strings, 65 + near-miss variants of them (case, hyphenation, whitespace, + no-underscore), and 8 controls that must stay low-risk -- 6 real + methods sharing a word with a high-risk one (`get_order_details` vs + `pay_order`, `list_disputes` vs `accept_dispute_claim`, ...) and 2 + explicitly synthetic names probing prefix matching. 0 false positives, + 0 false negatives. The prefix probe used to be the real + `send_invoice_reminder`; after the second widening made it high-risk, + no real low-risk method contains a high-risk method name as a + substring at all, so the property is probed synthetically and labelled + as such rather than quietly dropped. + Worth being precise about what this does and doesn't prove: `method` is + a closed, fixed dispatch string chosen by the calling framework's own + tool definitions, not attacker-controlled free text -- `PayPalAPI.run()` + itself already rejects any string not in the real 31-tool catalog. This + isn't the same class of finding as, say, a free-text query language + where a fragmented/concatenated value can evade a keyword scan; there's + no equivalent evasion surface here to find in the first place. + +**Also verified live against a real PayPal sandbox account** (see +`examples/tulip/datasets/live_sandbox.py`, credential-gated, not run in +CI) -- no mocks: a real `create_order` genuinely created a real sandbox +order over a real HTTPS call; `get_order_details` genuinely read it back; +`pay_order` was genuinely held and never reached PayPal; the same +`pay_order` call through a `GovernedPayPalAPI` constructed with an +explicit allow-everything policy override genuinely reached PayPal's real +API and got PayPal's own real `ORDER_NOT_APPROVED` business rejection +back -- proving the override is real, not a stub, and that this gate and +PayPal's own business rules are two independent, composable layers. Two +more real findings from that live run, neither a bug in this gate: +`get_order_details`'s own response message always says "has been +successfully captured" regardless of the order's real status (looks +copy-pasted from `capture_order`'s handler); `list_transactions` returned +a real `403 Forbidden` from PayPal's Transaction Search API on this +sandbox app, likely a scope this particular sandbox app doesn't have +enabled. This run also caught a real bug in an earlier draft of this +module's own examples: `app_agent.py`/the datasets above used the wrong +request field name (`id` instead of the real `order_id`, per +`shared/orders/parameters.py`'s `OrderIdParameters`/ +`CaptureOrderParameters`) -- invisible under full mocking (which replaces +`execute()` wholesale and never validates params against the real +schema), caught immediately once real schema validation was in the loop. +Fixed everywhere it appeared. + +The other 8 high-risk methods (`accept_dispute_claim`, +`cancel_subscription`, `cancel_sent_invoice`, `send_invoice`, +`create_subscription`, `create_subscription_plan`, +`create_recurring_series`, `activate_recurring_series`) remain verified +only via the mocked full-catalog sweep above, not live -- exercising most +of them for real needs pre-existing real sandbox state (an approved +subscription, a filed dispute) that itself requires a real +buyer-approval redirect flow, out of scope for this pass. Disclosed, not +glossed over. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +from collections.abc import Coroutine +from typing import Any + +from tulip.control import Action, AuditTrail, ControlPolicy, admit + +from ..shared.api import PayPalAPI + +# The thirteen tools with a real, hard-to-undo financial, liability, or +# real-external-party consequence in the current tool catalog -- see +# module docstring for why each one. +HIGH_RISK_METHODS = frozenset( + { + "pay_order", + "accept_dispute_claim", + "cancel_subscription", + "cancel_sent_invoice", + "send_invoice", + "create_subscription", + "create_subscription_plan", + "create_recurring_series", + "activate_recurring_series", + # Second widening. `send_invoice` was held on the reasoning that + # it is "a real external communication with real business/legal + # weight, not a draft" -- but the four below were left low-risk, + # and the ground truth filed them under + # "without-notifying-anyone", which for three of them is simply + # not true. See governance.py's docstring for the per-method + # argument and the one weak case. + "generate_invoice_qr_code", + "setup_invoice_auto_reminders", + "update_invoice_auto_reminder", + "send_invoice_reminder", + } +) + +# Two policies, chosen by whether a call was classified high-risk. A +# low-risk (read/draft) call auto-allows; a high-risk one is held for a +# human unless a caller supplies its own stricter/looser ControlPolicy. +_LOW_RISK_POLICY = ControlPolicy( + require_verification_score=0.0, + require_human_for=frozenset(), + max_blast_radius=10, +) +_HIGH_RISK_POLICY = ControlPolicy( + require_verification_score=0.0, + require_human_for=frozenset({"high-risk"}), + max_blast_radius=1, +) + + +def classify(method: str, params: dict[str, Any]) -> Action: + """Classifies one proposed PayPalAPI call as a tulip-agents Action. + + `params` is accepted (not just `method`) so a caller can build a + stricter classifier on top of this one -- e.g. amount-aware escalation + for `pay_order` once the order's amount has been looked up -- without + needing to change the call site. This module's own classifier is + method-only, per the amount-lookup gap disclosed above. + """ + is_high_risk = method in HIGH_RISK_METHODS + return Action( + name=method, + asset="paypal-account", + blast_radius=5 if is_high_risk else 1, + environment="production", + kind="paypal-financial-action" if is_high_risk else "paypal-read-or-draft", + tags=frozenset({"high-risk"}) if is_high_risk else frozenset(), + ) + + +class GovernedPayPalAPI(PayPalAPI): + """A drop-in `PayPalAPI` that gates every real call through `admit()`. + + Duck-type compatible with plain `PayPalAPI` -- anything that accepts a + `PayPalAPI` instance (this toolkit's own `openai.tool.PayPalTool`, + `langchain.tool.PayPalTool`, etc. all just call `.run(method, params)`) + accepts this instead with no other code changes. + + Note on scope: each existing framework `Toolkit.__init__` currently + constructs its own internal `PayPalAPI` rather than accepting one as a + constructor argument, so swapping in `GovernedPayPalAPI` for e.g. + `openai.toolkit.PayPalToolkit` today means calling that toolkit's own + `PayPalTool()` tool-factory function directly against a + `GovernedPayPalAPI` instance (see `examples/tulip/app_agent.py`) + rather than constructing `PayPalToolkit` itself. Making + `paypal_api` an optional constructor argument on the existing + toolkits would let this plug into all four uniformly -- a real, + small, complementary change this contribution doesn't make on its + own, since it touches files this change didn't otherwise need to. + """ + + def __init__( + self, + *args: Any, + policy: ControlPolicy | None = None, + trail: AuditTrail | None = None, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self._policy_override = policy + self._trail = trail if trail is not None else AuditTrail() + + def audit_trail(self) -> AuditTrail: + """The tamper-evident record of every decision this instance has + made (`.records()`, `.verify()`, `.export_jsonl()`).""" + return self._trail + + def run(self, method: str, params: dict) -> str: + """Same sync signature as `PayPalAPI.run()` -- every existing tool + wrapper in this toolkit calls `.run()` synchronously, so this stays + sync too rather than forcing every call site to change. `admit()` + itself is async, so this bridges to it; see `_run_admit_sync` for + why that bridge needs to handle both "no event loop running" and + "called from inside one" (e.g. the OpenAI Agents SDK's own + `on_invoke_tool` is itself a coroutine).""" + action = classify(method, params) + policy = self._policy_override or ( + _HIGH_RISK_POLICY if "high-risk" in action.tags else _LOW_RISK_POLICY + ) + + async def _perform() -> str: + return super(GovernedPayPalAPI, self).run(method, params) + + return _run_admit_sync( + admit(action, _perform, policy=policy, trail=self._trail) + ) + + +def _run_admit_sync(coro: Coroutine[Any, Any, str]) -> str: + """Runs an `admit()` coroutine to completion from sync code, whether or + not an event loop is already running on this thread. + + If nothing is running, `asyncio.run()` is enough. If a loop IS already + running on this thread (true whenever `GovernedPayPalAPI.run()` is + called from inside an async tool-invocation callback, e.g. the OpenAI + Agents SDK's `on_invoke_tool`), `asyncio.run()` would raise -- you + can't block-run a second loop on top of one that's already driving the + current call stack. Runs it on a fresh loop in a separate thread + instead, which is always safe regardless of the caller's own loop + state. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(asyncio.run, coro).result()