From 14cefcd4fc97a65976c8833e585c4e5dfe28361e Mon Sep 17 00:00:00 2001 From: Arwa Date: Tue, 21 Jan 2025 15:48:40 -0600 Subject: [PATCH 01/11] chore: Add support for Python 3.13 --- .github/workflows/unittest.yml | 2 +- CONTRIBUTING.rst | 10 ++++++---- noxfile.py | 4 ++-- owlbot.py | 4 ++-- setup.py | 1 + 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 8659d83d82..a7805de447 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python: ['3.9', '3.10', '3.11', '3.12'] + python: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8d68e4fc27..0a6dd06dc6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows. + 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -72,7 +72,7 @@ We use `nox `__ to instrument our tests. - To run a single unit test:: - $ nox -s unit-3.12 -- -k + $ nox -s unit-3.13 -- -k .. note:: @@ -143,12 +143,12 @@ Running System Tests $ nox -s system # Run a single system test - $ nox -s system-3.12 -- -k + $ nox -s system-3.13 -- -k .. note:: - System tests are only configured to run under Python 3.9, 3.11 and 3.12. + System tests are only configured to run under Python 3.9, 3.11, 3.12 and 3.13. For expediency, we do not run them in older versions of Python 3. This alone will not run the tests. You'll need to change some local @@ -262,11 +262,13 @@ We support: - `Python 3.10`_ - `Python 3.11`_ - `Python 3.12`_ +- `Python 3.13`_ .. _Python 3.9: https://docs.python.org/3.9/ .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ +.. _Python 3.12: https://docs.python.org/3.13/ Supported versions can be found in our ``noxfile.py`` `config`_. diff --git a/noxfile.py b/noxfile.py index 863c7b26d3..5f30609b31 100644 --- a/noxfile.py +++ b/noxfile.py @@ -47,7 +47,7 @@ DEFAULT_PYTHON_VERSION = "3.10" -UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +UNIT_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -65,7 +65,7 @@ # There are 4 different ibis-framework 9.x versions we want to test against. # 3.10 is needed for Windows tests. -SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.10", "3.12", "3.13"] SYSTEM_TEST_STANDARD_DEPENDENCIES = [ "jinja2", "mock", diff --git a/owlbot.py b/owlbot.py index 5de70bcad6..10fc47ebd7 100644 --- a/owlbot.py +++ b/owlbot.py @@ -31,8 +31,8 @@ # ---------------------------------------------------------------------------- templated_files = common.py_library( default_python_version="3.10", - unit_test_python_versions=["3.9", "3.10", "3.11", "3.12"], - system_test_python_versions=["3.9", "3.11", "3.12"], + unit_test_python_versions=["3.9", "3.10", "3.11", "3.12", "3.13"], + system_test_python_versions=["3.9", "3.11", "3.12", "3.13"], cov_level=35, intersphinx_dependencies={ "pandas": "https://pandas.pydata.org/pandas-docs/stable/", diff --git a/setup.py b/setup.py index 74a0d5475c..faf0251780 100644 --- a/setup.py +++ b/setup.py @@ -125,6 +125,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", "Topic :: Internet", ], From c998c116a235aa9c5a0f046d359e327737881769 Mon Sep 17 00:00:00 2001 From: Arwa Date: Wed, 22 Jan 2025 14:13:50 -0600 Subject: [PATCH 02/11] update contibuting.rst with the the new version --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0a6dd06dc6..5374e7e377 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -268,7 +268,7 @@ We support: .. _Python 3.10: https://docs.python.org/3.10/ .. _Python 3.11: https://docs.python.org/3.11/ .. _Python 3.12: https://docs.python.org/3.12/ -.. _Python 3.12: https://docs.python.org/3.13/ +.. _Python 3.13: https://docs.python.org/3.13/ Supported versions can be found in our ``noxfile.py`` `config`_. From 0a46d4a4ef78bef40c6a1eb597e001bfda4b9707 Mon Sep 17 00:00:00 2001 From: Arwa Date: Wed, 22 Jan 2025 15:04:01 -0600 Subject: [PATCH 03/11] skip sklearn import --- scripts/test_publish_api_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test_publish_api_coverage.py b/scripts/test_publish_api_coverage.py index 0b87563482..3f9f794c1d 100644 --- a/scripts/test_publish_api_coverage.py +++ b/scripts/test_publish_api_coverage.py @@ -26,6 +26,7 @@ def api_coverage_df(): def test_api_coverage_produces_expected_schema(api_coverage_df): + pytest.importorskip("sklearn") if sys.version.split(".")[:2] == ["3", "9"]: pytest.skip( "Python 3.9 uses older pandas without good microsecond timestamp support." @@ -55,5 +56,6 @@ def test_api_coverage_produces_expected_schema(api_coverage_df): def test_api_coverage_produces_missing_parameters(api_coverage_df): + pytest.importorskip("sklearn") """Make sure at least some functions have reported missing parameters.""" assert (api_coverage_df["missing_parameters"].str.len() > 0).any() From 4c9e4a64955e97f18faf2f6e7c285a9634f795ea Mon Sep 17 00:00:00 2001 From: Arwa Date: Fri, 24 Jan 2025 10:38:27 -0600 Subject: [PATCH 04/11] skip api coverage tests because of sklearn NameError --- scripts/test_publish_api_coverage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/test_publish_api_coverage.py b/scripts/test_publish_api_coverage.py index 3f9f794c1d..bfa17981c3 100644 --- a/scripts/test_publish_api_coverage.py +++ b/scripts/test_publish_api_coverage.py @@ -25,8 +25,11 @@ def api_coverage_df(): return publish_api_coverage.build_api_coverage_table("my_bf_ver", "my_release_ver") +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for these test in python 3.13", +) def test_api_coverage_produces_expected_schema(api_coverage_df): - pytest.importorskip("sklearn") if sys.version.split(".")[:2] == ["3", "9"]: pytest.skip( "Python 3.9 uses older pandas without good microsecond timestamp support." @@ -55,7 +58,10 @@ def test_api_coverage_produces_expected_schema(api_coverage_df): ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for these test in python 3.13", +) def test_api_coverage_produces_missing_parameters(api_coverage_df): - pytest.importorskip("sklearn") """Make sure at least some functions have reported missing parameters.""" assert (api_coverage_df["missing_parameters"].str.len() > 0).any() From 678b676d113d128e312265f78d7d4f78180eb5f5 Mon Sep 17 00:00:00 2001 From: Arwa Date: Fri, 24 Jan 2025 14:59:26 -0600 Subject: [PATCH 05/11] skip remote function tests --- scripts/test_publish_api_coverage.py | 4 ++-- tests/system/large/test_remote_function.py | 25 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/test_publish_api_coverage.py b/scripts/test_publish_api_coverage.py index bfa17981c3..034a266177 100644 --- a/scripts/test_publish_api_coverage.py +++ b/scripts/test_publish_api_coverage.py @@ -27,7 +27,7 @@ def api_coverage_df(): @pytest.mark.skipif( sys.version_info >= (3, 13), - reason="Issues with installing sklearn for these test in python 3.13", + reason="Issues with installing sklearn for this test in python 3.13", ) def test_api_coverage_produces_expected_schema(api_coverage_df): if sys.version.split(".")[:2] == ["3", "9"]: @@ -60,7 +60,7 @@ def test_api_coverage_produces_expected_schema(api_coverage_df): @pytest.mark.skipif( sys.version_info >= (3, 13), - reason="Issues with installing sklearn for these test in python 3.13", + reason="Issues with installing sklearn for this test in python 3.13", ) def test_api_coverage_produces_missing_parameters(api_coverage_df): """Make sure at least some functions have reported missing parameters.""" diff --git a/tests/system/large/test_remote_function.py b/tests/system/large/test_remote_function.py index d0eb6c1904..159823dbab 100644 --- a/tests/system/large/test_remote_function.py +++ b/tests/system/large/test_remote_function.py @@ -18,6 +18,7 @@ import math # must keep this at top level to test udf referring global import import os.path import shutil +import sys import tempfile import textwrap @@ -881,6 +882,10 @@ def test_remote_udf_lambda(session, scalars_dfs, dataset_id, bq_cf_connection): ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_explicit_name( session, scalars_dfs, dataset_id, bq_cf_connection @@ -938,6 +943,10 @@ def square(x): ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_external_package_dependencies( session, scalars_dfs, dataset_id, bq_cf_connection @@ -983,6 +992,10 @@ def pd_np_foo(x): ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_explicit_name_reuse( session, scalars_dfs, dataset_id, bq_cf_connection @@ -1354,6 +1367,10 @@ def square_num(x): ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_gcf_cmek(): # TODO(shobs): Automate the following set-up during testing in the test project. @@ -2409,6 +2426,10 @@ def square(x: int) -> int: return x * x +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.parametrize( ("session_creator"), [ @@ -2484,6 +2505,10 @@ def add_one(x: int) -> int: ) +@pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Issues with installing sklearn for this test in python 3.13", +) @pytest.mark.parametrize( ("session_creator"), [ From 25bc7383c43a0461712e68d4c696ea5498cbb2f2 Mon Sep 17 00:00:00 2001 From: Arwa Date: Fri, 24 Jan 2025 16:45:27 -0600 Subject: [PATCH 06/11] skip all remote function tests and doctests --- bigframes/session/__init__.py | 3 +- tests/system/large/test_remote_function.py | 30 ++++--------------- .../bigframes_vendored/pandas/core/frame.py | 2 ++ .../bigframes_vendored/pandas/core/series.py | 7 +++-- 4 files changed, 15 insertions(+), 27 deletions(-) diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 1d85967729..6dd6c6cc50 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -1419,6 +1419,7 @@ def read_gbq_function( (See: https://github.com/GoogleCloudPlatform/bigquery-utils/tree/master/udfs/community#community-udfs). **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -1482,7 +1483,7 @@ def read_gbq_function( 2 TestCad$123456Str dtype: string - Another use case is to define your own remote funtion and use it later. + Another use case is to define your own remote function and use it later. For example, define the remote function: >>> @bpd.remote_function() diff --git a/tests/system/large/test_remote_function.py b/tests/system/large/test_remote_function.py index 159823dbab..0532f58a4c 100644 --- a/tests/system/large/test_remote_function.py +++ b/tests/system/large/test_remote_function.py @@ -48,6 +48,12 @@ _team_euler = "Team Euler" +pytestmark = pytest.mark.skipif( + sys.version_info >= (3, 13), + reason="Runtime 'python313' is not supported yet. Skip for now.", +) + + def cleanup_remote_function_assets( bigquery_client, cloudfunctions_client, remote_udf, ignore_failures=True ): @@ -882,10 +888,6 @@ def test_remote_udf_lambda(session, scalars_dfs, dataset_id, bq_cf_connection): ) -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_explicit_name( session, scalars_dfs, dataset_id, bq_cf_connection @@ -943,10 +945,6 @@ def square(x): ) -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_external_package_dependencies( session, scalars_dfs, dataset_id, bq_cf_connection @@ -992,10 +990,6 @@ def pd_np_foo(x): ) -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_explicit_name_reuse( session, scalars_dfs, dataset_id, bq_cf_connection @@ -1367,10 +1361,6 @@ def square_num(x): ) -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.flaky(retries=2, delay=120) def test_remote_function_with_gcf_cmek(): # TODO(shobs): Automate the following set-up during testing in the test project. @@ -2426,10 +2416,6 @@ def square(x: int) -> int: return x * x -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.parametrize( ("session_creator"), [ @@ -2505,10 +2491,6 @@ def add_one(x: int) -> int: ) -@pytest.mark.skipif( - sys.version_info >= (3, 13), - reason="Issues with installing sklearn for this test in python 3.13", -) @pytest.mark.parametrize( ("session_creator"), [ diff --git a/third_party/bigframes_vendored/pandas/core/frame.py b/third_party/bigframes_vendored/pandas/core/frame.py index f5aa23d00b..1367fce0f3 100644 --- a/third_party/bigframes_vendored/pandas/core/frame.py +++ b/third_party/bigframes_vendored/pandas/core/frame.py @@ -4358,6 +4358,7 @@ def map(self, func, na_action: Optional[str] = None) -> DataFrame: DataFrame.map. **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -4718,6 +4719,7 @@ def apply(self, func, *, axis=0, args=(), **kwargs): ``axis=1`` scenario is in preview. **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> import pandas as pd diff --git a/third_party/bigframes_vendored/pandas/core/series.py b/third_party/bigframes_vendored/pandas/core/series.py index 727e25836a..cb02548c99 100644 --- a/third_party/bigframes_vendored/pandas/core/series.py +++ b/third_party/bigframes_vendored/pandas/core/series.py @@ -1772,6 +1772,7 @@ def apply( python function then converting it into a `remote_function` is recommended. **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -4935,6 +4936,7 @@ def mask(self, cond, other): """Replace values where the condition is True. **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -5038,13 +5040,13 @@ def mask(self, cond, other): with corresponding value from other. If cond is callable, it is computed on the Series/DataFrame and should return boolean Series/DataFrame or array. The callable must not change input - Series/DataFrame (though pandas doesn’t check it). + Series/DataFrame (though pandas doesn't check it). other (scalar, Series/DataFrame, or callable): Entries where cond is True are replaced with corresponding value from other. If other is callable, it is computed on the Series/DataFrame and should return scalar or Series/DataFrame. The callable must not change input Series/DataFrame (though pandas - doesn’t check it). If not specified, entries will be filled with + doesn't check it). If not specified, entries will be filled with the corresponding NULL value (np.nan for numpy dtypes, pd.NA for extension dtypes). @@ -5554,6 +5556,7 @@ def map( are treated the same as ``dict``. **Examples:** + .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None From 106af9a574a51a4611073ba608b0d6a07829f665 Mon Sep 17 00:00:00 2001 From: Arwa Date: Tue, 28 Jan 2025 10:34:02 -0600 Subject: [PATCH 07/11] update doctest python version to 3.12 in noxfile --- bigframes/session/__init__.py | 1 - noxfile.py | 15 ++++----------- .../bigframes_vendored/pandas/core/frame.py | 2 -- .../bigframes_vendored/pandas/core/series.py | 3 --- 4 files changed, 4 insertions(+), 17 deletions(-) diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 6dd6c6cc50..4474e791dd 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -1419,7 +1419,6 @@ def read_gbq_function( (See: https://github.com/GoogleCloudPlatform/bigquery-utils/tree/master/udfs/community#community-udfs). **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None diff --git a/noxfile.py b/noxfile.py index 5f30609b31..7c8d0e3f82 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,7 +24,6 @@ import shutil import time from typing import Dict, List -import warnings import nox import nox.sessions @@ -32,6 +31,9 @@ BLACK_VERSION = "black==22.3.0" ISORT_VERSION = "isort==5.12.0" +# TODO: switch to 3.13 once remote functions / cloud run adds a runtime for it (internal issue 333742751) +LATEST_FULLY_SUPPORTED_PYTHON = "3.12" + # pytest-retry is not yet compatible with pytest 8.x. # https://github.com/str0zzapreti/pytest-retry/issues/32 PYTEST_VERSION = "pytest<8.0.0dev" @@ -57,7 +59,6 @@ "pytest-asyncio", "pytest-mock", ] -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] UNIT_TEST_DEPENDENCIES: List[str] = [] UNIT_TEST_EXTRAS: List[str] = [] @@ -169,14 +170,6 @@ def install_unittest_dependencies(session, install_test_extra, *constraints): standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES session.install(*standard_deps, *constraints) - if UNIT_TEST_EXTERNAL_DEPENDENCIES: - msg = ( - "'unit_test_external_dependencies' is deprecated. Instead, please " - "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", - ) - warnings.warn(msg, DeprecationWarning) - session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) - if UNIT_TEST_LOCAL_DEPENDENCIES: session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) @@ -386,7 +379,7 @@ def system_noextras(session: nox.sessions.Session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@nox.session(python=LATEST_FULLY_SUPPORTED_PYTHON) def doctest(session: nox.sessions.Session): """Run the system test suite.""" run_system( diff --git a/third_party/bigframes_vendored/pandas/core/frame.py b/third_party/bigframes_vendored/pandas/core/frame.py index 1367fce0f3..f5aa23d00b 100644 --- a/third_party/bigframes_vendored/pandas/core/frame.py +++ b/third_party/bigframes_vendored/pandas/core/frame.py @@ -4358,7 +4358,6 @@ def map(self, func, na_action: Optional[str] = None) -> DataFrame: DataFrame.map. **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -4719,7 +4718,6 @@ def apply(self, func, *, axis=0, args=(), **kwargs): ``axis=1`` scenario is in preview. **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> import pandas as pd diff --git a/third_party/bigframes_vendored/pandas/core/series.py b/third_party/bigframes_vendored/pandas/core/series.py index cb02548c99..e38c671bdd 100644 --- a/third_party/bigframes_vendored/pandas/core/series.py +++ b/third_party/bigframes_vendored/pandas/core/series.py @@ -1772,7 +1772,6 @@ def apply( python function then converting it into a `remote_function` is recommended. **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -4936,7 +4935,6 @@ def mask(self, cond, other): """Replace values where the condition is True. **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None @@ -5556,7 +5554,6 @@ def map( are treated the same as ``dict``. **Examples:** - .. doctest-skip-all >>> import bigframes.pandas as bpd >>> bpd.options.display.progress_bar = None From 57f72270237ef0db00eb76f1349706d132ad3f4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20Swe=C3=B1a=20=28Swast=29?= Date: Tue, 28 Jan 2025 12:34:51 -0600 Subject: [PATCH 08/11] Update noxfile.py --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 7c8d0e3f82..191c3d6944 100644 --- a/noxfile.py +++ b/noxfile.py @@ -368,7 +368,7 @@ def system(session: nox.sessions.Session): ) -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) +@nox.session(python=LATEST_FULLY_SUPPORTED_PYTHON) def system_noextras(session: nox.sessions.Session): """Run the system test suite.""" run_system( From 366e4f889a70995c176e1fe75aaa557fb357cf86 Mon Sep 17 00:00:00 2001 From: Arwa Date: Tue, 28 Jan 2025 15:37:12 -0600 Subject: [PATCH 09/11] fix the system_pre-release error --- bigframes/core/expression.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bigframes/core/expression.py b/bigframes/core/expression.py index 9173bebfc4..3ffccc94ac 100644 --- a/bigframes/core/expression.py +++ b/bigframes/core/expression.py @@ -20,6 +20,8 @@ import typing from typing import Mapping, TypeVar, Union +import pandas as pd + import bigframes.core.identifiers as ids import bigframes.dtypes as dtypes import bigframes.operations @@ -253,6 +255,17 @@ def is_bijective(self) -> bool: # () <-> value return True + def __eq__(self, other): + if not isinstance(other, ScalarConstantExpression): + return False + + # With python 3.13 and the pre-release version of pandas, + # NA == NA is NA instead of True + if pd.isna(self.value) and pd.isna(other.value): # type: ignore + return self.dtype == other.dtype + + return self.value == other.value and self.dtype == other.dtype + @dataclasses.dataclass(frozen=True) class UnboundVariableExpression(Expression): From 66f584d0c44ec01c34740a8507ba450b3833e5b0 Mon Sep 17 00:00:00 2001 From: Tim Swena Date: Wed, 29 Jan 2025 10:46:10 -0600 Subject: [PATCH 10/11] skip remote functions notebooks on 3.13 tests --- .gitignore | 7 +++ .../getting_started_bq_dataframes.ipynb | 14 ++++++ notebooks/location/regionalized.ipynb | 14 ++++++ .../remote_functions/remote_function.ipynb | 15 ++++++ .../remote_function_usecases.ipynb | 14 ++++++ .../remote_function_vertex_claude_model.ipynb | 14 ++++++ noxfile.py | 49 +++++++++++++------ 7 files changed, 113 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index d083ea1ddc..c35ad2d31b 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,10 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc pylintrc.test + +# Notebook tests +*.backup +*.bq_exec_time_seconds +*.bytesprocessed +*.error +*.slotmillis diff --git a/notebooks/getting_started/getting_started_bq_dataframes.ipynb b/notebooks/getting_started/getting_started_bq_dataframes.ipynb index 38ce75cc25..c5deeef1c5 100644 --- a/notebooks/getting_started/getting_started_bq_dataframes.ipynb +++ b/notebooks/getting_started/getting_started_bq_dataframes.ipynb @@ -1448,6 +1448,20 @@ "Running your own Python functions (or being able to bring your packages) and using them at scale is a challenge many data scientists face. BigQuery DataFrames makes it easy to deploy [remote functions](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.pandas#bigframes_pandas_remote_function) that run scalar Python functions at BigQuery scale. These functions are persisted as [BigQuery remote functions](https://cloud.google.com/bigquery/docs/remote-functions) that you can then re-use." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/notebooks/location/regionalized.ipynb b/notebooks/location/regionalized.ipynb index 5a8239a42a..1b138c6a66 100644 --- a/notebooks/location/regionalized.ipynb +++ b/notebooks/location/regionalized.ipynb @@ -1339,6 +1339,20 @@ "# Using the Remote Functions" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "attachments": {}, "cell_type": "markdown", diff --git a/notebooks/remote_functions/remote_function.ipynb b/notebooks/remote_functions/remote_function.ipynb index 1c1048d356..2114311e10 100644 --- a/notebooks/remote_functions/remote_function.ipynb +++ b/notebooks/remote_functions/remote_function.ipynb @@ -1,5 +1,20 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "bcff4fc4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "code", "execution_count": 19, diff --git a/notebooks/remote_functions/remote_function_usecases.ipynb b/notebooks/remote_functions/remote_function_usecases.ipynb index b897def4e8..d4dde6e6b1 100644 --- a/notebooks/remote_functions/remote_function_usecases.ipynb +++ b/notebooks/remote_functions/remote_function_usecases.ipynb @@ -21,6 +21,20 @@ "# limitations under the License." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb index 641a30e104..78f0d27474 100644 --- a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb +++ b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb @@ -28,6 +28,20 @@ "" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "# Python 3.13 is not yet a supported runtime for remote functions.\n", + "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", + "if sys.version_info >= (3, 13, 0):\n", + " sys.exit(0)" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/noxfile.py b/noxfile.py index 74ee729d28..38e5fab1a6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -755,6 +755,20 @@ def notebook(session: nox.Session): "notebooks/apps/synthetic_data_generation.ipynb", ] + # TODO: remove exception for Python 3.13 cloud run adds a runtime for it (internal issue 333742751) + # TODO: remove exception for Python 3.13 if nbmake adds support for + # sys.exit(0) or pytest.skip(...). + # See: https://github.com/treebeardtech/nbmake/issues/134 + if session.python == "3.13": + denylist.extend( + [ + "notebooks/getting_started/getting_started_bq_dataframes.ipynb", + "notebooks/remote_functions/remote_function_usecases.ipynb", + "notebooks/remote_functions/remote_function_vertex_claude_model.ipynb", + "notebooks/remote_functions/remote_function.ipynb", + ] + ) + # Convert each Path notebook object to a string using a list comprehension. notebooks = [str(nb) for nb in notebooks_list] @@ -762,20 +776,27 @@ def notebook(session: nox.Session): notebooks = list(filter(lambda nb: nb not in denylist, notebooks)) # Regionalized notebooks - notebooks_reg = { - "regionalized.ipynb": [ - "asia-southeast1", - "eu", - "europe-west4", - "southamerica-west1", - "us", - "us-central1", - ] - } - notebooks_reg = { - os.path.join("notebooks/location", nb): regions - for nb, regions in notebooks_reg.items() - } + # TODO: remove exception for Python 3.13 cloud run adds a runtime for it (internal issue 333742751) + # TODO: remove exception for Python 3.13 if nbmake adds support for + # sys.exit(0) or pytest.skip(...). + # See: https://github.com/treebeardtech/nbmake/issues/134 + if session.python == "3.13": + notebooks_reg = {} + else: + notebooks_reg = { + "regionalized.ipynb": [ + "asia-southeast1", + "eu", + "europe-west4", + "southamerica-west1", + "us", + "us-central1", + ] + } + notebooks_reg = { + os.path.join("notebooks/location", nb): regions + for nb, regions in notebooks_reg.items() + } # The pytest --nbmake exits silently with "no tests ran" message if # one of the notebook paths supplied does not exist. Let's make sure that From 770fef2dd74510795dc2f6d8bfd33e2dc740b47f Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 29 Jan 2025 16:48:30 +0000 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .gitignore | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.gitignore b/.gitignore index c35ad2d31b..d083ea1ddc 100644 --- a/.gitignore +++ b/.gitignore @@ -62,10 +62,3 @@ system_tests/local_test_setup # Make sure a generated file isn't accidentally committed. pylintrc pylintrc.test - -# Notebook tests -*.backup -*.bq_exec_time_seconds -*.bytesprocessed -*.error -*.slotmillis