IBM · shinnar · Apr 4, 2025 · Apr 2, 2025 · Apr 3, 2025 · Apr 3, 2025
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -158,7 +158,7 @@ jobs:
           python-version: 3.9
           setup-target: '.[full,test]'
         - test-case: test/test_relational_sklearn.py
-          python-version: 3.9
+          python-version: '3.11'
           setup-target: '.[full,test]'
 
     steps:
@@ -242,7 +242,7 @@ jobs:
           python-version: 3.9
           setup-target: '.[full,test]'
         - test-case: test/test_relational_sklearn.py
-          python-version: 3.9
+          python-version: '3.11'
           setup-target: '.[full,test]'
         - test-case: test/test_snapml.py
           python-version: 3.9

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Lale
 
-[![Tests](https://github.com/IBM/lale/workflows/Tests/badge.svg?branch=master)](https://github.com/IBM/lale/actions?query=workflow%3ATests+branch%3Amaster)
+[![Tests](https://github.com/IBM/lale/actions/workflows/build.yml/badge.svg)](https://github.com/IBM/lale/actions/workflows/build.yml)
 [![Documentation Status](https://readthedocs.org/projects/lale/badge/?version=latest)](https://lale.readthedocs.io/en/latest/?badge=latest)
 [![PyPI version shields.io](https://img.shields.io/pypi/v/lale?color=success)](https://pypi.python.org/pypi/lale/)
 [![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/)

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -6,7 +6,7 @@ graphviz
 hyperopt
 jsonschema
 jsonsubschema
-scikit-learn>=1.0.0,<1.5.0
+scikit-learn>=1.0.0,<1.7.0
 scipy
 pandas
 decorator
@@ -16,7 +16,7 @@ typing-extensions
 # subset of setup.py -> extras_require -> full
 # mystic
 xgboost
-lightgbm<4.4.0
+lightgbm<4.7.0
 liac-arff>=2.4.0
 imbalanced-learn
 smac<=0.10.0

diff --git a/examples/docs_guide_for_sklearn_users.ipynb b/examples/docs_guide_for_sklearn_users.ipynb
@@ -1502,7 +1502,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert message.startswith(\"Negative values in data passed to NMF (input X)\")"
+    "assert message.startswith(\"Negative values in data passed to NMF\")"
    ]
   },
   {
@@ -1735,7 +1735,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "lale311",
    "language": "python",
    "name": "python3"
   },
@@ -1749,7 +1749,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.11.1"
   }
  },
  "nbformat": 4,

diff --git a/lale/lib/_common_schemas.py b/lale/lib/_common_schemas.py
@@ -48,6 +48,7 @@
 .. _`lale.lib.aif360 metrics`: https://lale.readthedocs.io/en/latest/modules/lale.lib.aif360.html#metrics
 """,
             "laleType": "callable",
+            "forOptimizer": False,
         },
         {
             "description": "Known scorer for classification task.",

diff --git a/lale/lib/autogen/mini_batch_sparse_pca.py b/lale/lib/autogen/mini_batch_sparse_pca.py
@@ -91,7 +91,10 @@ def inverse_transform(self, X):
                     "description": "number of iterations to perform for each mini batch",
                 },
                 "callback": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "callable that gets invoked every five iterations",
                 },

diff --git a/lale/lib/autogen/radius_neighbors_classifier.py b/lale/lib/autogen/radius_neighbors_classifier.py
@@ -196,6 +196,7 @@ def predict(self, X):
                 {"enum": ["euclidean", "manhattan", "minkowski", "nan_euclidean"]},
                 {
                     "laleType": "callable",
+                    "forOptimizer": False,
                     "description": "Takes two arrays representing 1D vectors as inputs and must return one value indicating the distance between those vectors.",
                 },
             ],

diff --git a/lale/lib/autogen/ransac_regressor.py b/lale/lib/autogen/ransac_regressor.py
@@ -77,12 +77,18 @@ def predict(self, X):
                     "description": "Maximum residual for a data sample to be classified as an inlier",
                 },
                 "is_data_valid": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "This function is called with the randomly selected data before the model is fitted to it: `is_data_valid(X, y)`",
                 },
                 "is_model_valid": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "This function is called with the estimated model and the randomly selected data: `is_model_valid(model, X, y)`",
                 },

diff --git a/lale/lib/autogen/theil_sen_regressor.py b/lale/lib/autogen/theil_sen_regressor.py
@@ -177,7 +177,7 @@ def predict(self, X):
 
 if lale.operators.sklearn_version >= version.Version("1.6"):
     TheilSenRegressor = TheilSenRegressor.customize_schema(
-        copy={
+        copy_X={
             "anyOf": [{"type": "boolean"}, {"enum": ["deprecated"]}],
             "default": "deprecated",
             "description": "Deprecated.  Has not effect.",
@@ -187,7 +187,7 @@ def predict(self, X):
 
 if lale.operators.sklearn_version >= version.Version("1.8"):
     TheilSenRegressor = TheilSenRegressor.customize_schema(
-        copy=None, set_as_available=True
+        copy_X=None, set_as_available=True
     )
 
 set_docstrings(TheilSenRegressor)
diff --git a/lale/lib/lale/tee.py b/lale/lib/lale/tee.py
@@ -47,6 +47,7 @@ def transform_schema(self, s_X):
                             "is called on this operator, the callable will be passed the given"
                             "X and y values",
                             "laleType": "callable",
+                            "forOptimizer": False,
                         },
                         {
                             "description": "No listener.  Causes this operator to behave like NoOp.",

diff --git a/lale/lib/lightgbm/lgbm_classifier.py b/lale/lib/lightgbm/lgbm_classifier.py
@@ -361,7 +361,7 @@ def score(self, X, y):
             "anyOf": [
                 {"type": "array", "items": {"type": "string"}},
                 {"enum": ["logloss", None]},
-                {"laleType": "callable"},
+                {"laleType": "callable", "forOptimizer": False},
             ],
             "default": None,
             "description": "string, list of strings, callable or None, optional (default=None).",

diff --git a/lale/lib/lightgbm/lgbm_regressor.py b/lale/lib/lightgbm/lgbm_regressor.py
@@ -338,7 +338,7 @@ def score(self, X, y):
             "anyOf": [
                 {"type": "array", "items": {"type": "string"}},
                 {"enum": ["l2", None]},
-                {"laleType": "callable"},
+                {"laleType": "callable", "forOptimizer": False},
             ],
             "default": None,
             "description": "string, list of strings, callable or None, optional (default=None).",

diff --git a/lale/lib/rasl/batching.py b/lale/lib/rasl/batching.py
@@ -311,12 +311,18 @@ def predict(self, X, y=None):
                     "description": "Amount of memory to be used in bytes.",
                 },
                 "scoring": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "Batch-wise scoring metrics from `lale.lib.rasl`.",
                 },
                 "progress_callback": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "Callback function to get performance metrics per batch.",
                 },

diff --git a/lale/lib/rasl/project.py b/lale/lib/rasl/project.py
@@ -396,6 +396,7 @@ def is_keeper(column_schema):
                         },
                         {
                             "laleType": "callable",
+                            "forOptimizer": False,
                             "description": "Callable that is passed the input data X and can return a list of column names or indices.",
                         },
                         {
@@ -428,6 +429,7 @@ def is_keeper(column_schema):
                         },
                         {
                             "laleType": "callable",
+                            "forOptimizer": False,
                             "description": "Callable that is passed the input data X and can return a list of column names or indices.",
                         },
                         {

diff --git a/lale/lib/sklearn/column_transformer.py b/lale/lib/sklearn/column_transformer.py
@@ -77,6 +77,7 @@
                                     },
                                     {
                                         "laleType": "callable",
+                                        "forOptimizer": False,
                                         "not": {"type": ["integer", "array", "string"]},
                                         "description": "Callable that is passed the input data X and can return any of the above.",
                                     },
@@ -241,12 +242,13 @@
                     "type": "boolean",
                 },
                 {
-                    "type": "str",
+                    "type": "string",
                     "description": 'A string ready for formatting. The given string will be formatted using two field names: transformer_name and feature_name. e.g. "{feature_name}__{transformer_name}"',
                 },
                 {
                     "laleType": "callable",
                     "description": "A Callable[[str, str], str]. ColumnTransformer.get_feature_names_out will rename all the features using the name of the transformer. The first argument of the callable is the transformer name and the second argument is the feature name. The returned string will be the new feature name.",
+                    "forOptimizer": False,
                 },
             ],
             "default": True,

diff --git a/lale/lib/sklearn/function_transformer.py b/lale/lib/sklearn/function_transformer.py
@@ -38,12 +38,18 @@
             "relevantToOptimizer": [],
             "properties": {
                 "func": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "The callable to use for the transformation.",
                 },
                 "inverse_func": {
-                    "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+                    "anyOf": [
+                        {"laleType": "callable", "forOptimizer": False},
+                        {"enum": [None]},
+                    ],
                     "default": None,
                     "description": "The callable to use for the inverse transformation.",
                 },
@@ -166,7 +172,10 @@
     # new: https://scikit-learn.org/1.1/modules/generated/sklearn.preprocessing.FunctionTransformer.html
     FunctionTransformer = FunctionTransformer.customize_schema(
         feature_names_out={
-            "anyOf": [{"laleType": "callable"}, {"enum": ["one-to-one", None]}],
+            "anyOf": [
+                {"laleType": "callable", "forOptimizer": False},
+                {"enum": ["one-to-one", None]},
+            ],
             "default": None,
             "description": "Determines the list of feature names that will be returned by the ``get_feature_names_out`` method. If it is ‘one-to-one’, then the output feature names will be equal to the input feature names. If it is a callable, then it must take two positional arguments: this ``FunctionTransformer`` (``self``) and an array-like of input feature names (``input_features``). It must return an array-like of output feature names. The ``get_feature_names_out`` method is only defined if ``feature_names_out`` is not None.",
         },

diff --git a/lale/lib/sklearn/gradient_boosting_classifier.py b/lale/lib/sklearn/gradient_boosting_classifier.py
@@ -272,7 +272,10 @@
             "description": "Sample weights. If None, then samples are equally weighted. Splits",
         },
         "monitor": {
-            "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+            "anyOf": [
+                {"laleType": "callable", "forOptimizer": False},
+                {"enum": [None]},
+            ],
             "default": None,
             "description": "The monitor is called after each iteration with the current the current iteration, a reference to the estimator and the local variables of _fit_stages as keyword arguments callable(i, self, locals()).",
         },

diff --git a/lale/lib/sklearn/gradient_boosting_regressor.py b/lale/lib/sklearn/gradient_boosting_regressor.py
@@ -278,7 +278,10 @@
             "description": "Sample weights. If None, then samples are equally weighted. Splits",
         },
         "monitor": {
-            "anyOf": [{"laleType": "callable"}, {"enum": [None]}],
+            "anyOf": [
+                {"laleType": "callable", "forOptimizer": False},
+                {"enum": [None]},
+            ],
             "default": None,
             "description": "The monitor is called after each iteration with the current the current iteration, a reference to the estimator and the local variables of _fit_stages as keyword arguments callable(i, self, locals()).",
         },

diff --git a/lale/lib/sklearn/k_neighbors_classifier.py b/lale/lib/sklearn/k_neighbors_classifier.py
@@ -238,7 +238,8 @@
     KNeighborsClassifier = KNeighborsClassifier.customize_schema(
         metric={
             "anyOf": [
-                {"enum": ["euclidean", "manhattan", "minkowski", "nan_euclidean"]},
+                {"enum": ["euclidean", "manhattan", "minkowski"]},
+                {"type": "string", "forOptimizer": False},
                 {
                     "laleType": "callable",
                     "forOptimizer": False,

diff --git a/lale/lib/sklearn/pipeline.py b/lale/lib/sklearn/pipeline.py
@@ -221,5 +221,27 @@ def viz_label(self) -> str:
         ),
     )
 
+if lale.operators.sklearn_version >= version.Version("1.6"):
+    Pipeline = typing.cast(
+        lale.operators.PlannedIndividualOp,
+        Pipeline.customize_schema(
+            transform_input={
+                "anyOf": [
+                    {"enum": [None]},
+                    {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "forOptimizer": False,
+                    },
+                ],
+                "description": """
+The names of the metadata parameters that should be transformed by the pipeline before passing it to the step consuming it.
+This enables transforming some input arguments to fit (other than X) to be transformed by the steps of the pipeline up to the step which requires them. Requirement is defined via metadata routing. For instance, this can be used to pass a validation set through the pipeline.
+You can only set this if metadata routing is enabled, which you can enable using sklearn.set_config(enable_metadata_routing=True).""",
+                "default": None,
+            },
+            set_as_available=True,
+        ),
+    )
 
 lale.docstrings.set_docstrings(Pipeline)
diff --git a/lale/lib/sklearn/rfe.py b/lale/lib/sklearn/rfe.py
@@ -174,6 +174,7 @@
                 {
                     "description": "The callable is passed with the fitted estimator and it should return importance for each feature.",
                     "laleType": "callable",
+                    "forOptimizer": False,
                 },
             ],
             "default": "auto",

diff --git a/lale/lib/sklearn/select_k_best.py b/lale/lib/sklearn/select_k_best.py
@@ -50,6 +50,7 @@ def transform(self, X):
             "properties": {
                 "score_func": {
                     "laleType": "callable",
+                    "forOptimizer": False,
                     "default": sklearn.feature_selection.f_classif,
                     "description": "Function taking two arrays X and y, and returning a pair of arrays (scores, pvalues) or a single array with scores.",
                 },

diff --git a/lale/lib/xgboost/xgb_classifier.py b/lale/lib/xgboost/xgb_classifier.py
@@ -214,7 +214,7 @@ def score(self, X, y):
                                 "multi:softmax",
                             ]
                         },
-                        {"laleType": "callable"},
+                        {"laleType": "callable", "forOptimizer": False},
                     ],
                     "default": "binary:logistic",
                 },
@@ -562,7 +562,7 @@ def score(self, X, y):
                             "multi:softprob",
                         ]
                     },
-                    {"laleType": "callable"},
+                    {"laleType": "callable", "forOptimizer": False},
                 ],
                 "default": "binary:logistic",
             }
@@ -878,7 +878,11 @@ def score(self, X, y):
             "anyOf": [
                 {"type": "string"},
                 {"type": "array", "items": {"type": "string"}},
-                {"type": "array", "items": {"laleType": "callable"}},
+                {
+                    "type": "array",
+                    "items": {"laleType": "callable"},
+                    "forOptimizer": False,
+                },
                 {"enum": [None]},
             ],
             "default": None,
@@ -897,7 +901,11 @@ def score(self, X, y):
             "description": """List of callback functions that are applied at end of each iteration.
             It is possible to use predefined callbacks by using Callback API.""",
             "anyOf": [
-                {"type": "array", "items": {"laleType": "callable"}},
+                {
+                    "type": "array",
+                    "items": {"laleType": "callable"},
+                    "forOptimizer": False,
+                },
                 {"enum": [None]},
             ],
             "default": None,