diff --git a/.fvmrc b/.fvmrc index 5913bec0..4b28f0a5 100644 --- a/.fvmrc +++ b/.fvmrc @@ -1,3 +1,3 @@ { - "flutter": "3.44.4" + "flutter": "3.44.8" } \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ff86017..64512d60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,7 +102,7 @@ jobs: restore-keys: | flet-cache-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python_version }}- flet-cache-${{ runner.os }}-${{ runner.arch }}- - # Read-only on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Configure ${{ matrix.build_system }} @@ -170,7 +170,7 @@ jobs: restore-keys: | flet-cache-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python_version }}- flet-cache-${{ runner.os }}-${{ runner.arch }}- - # Read-only on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Setup iOS Simulator @@ -254,7 +254,7 @@ jobs: restore-keys: | flet-cache-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python_version }}- flet-cache-${{ runner.os }}-${{ runner.arch }}- - # Read-only on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Enable KVM @@ -270,7 +270,7 @@ jobs: # v5 was the last fully MIT-licensed major and is on Node 24. uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 with: - # Disable the Gradle cache on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. cache-disabled: ${{ startsWith(github.ref, 'refs/tags/') }} - name: AVD cache @@ -281,7 +281,7 @@ jobs: ~/.android/avd/* ~/.android/adb* key: avd-bridge - # Read-only on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Setup Android Emulator + Run tests @@ -362,7 +362,7 @@ jobs: restore-keys: | flet-cache-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python_version }}- flet-cache-${{ runner.os }}-${{ runner.arch }}- - # Read-only on tag/release builds to avoid cache poisoning. + # Avoid release-time cache poisoning. lookup-only: ${{ startsWith(github.ref, 'refs/tags/') }} - name: Package + run integration test @@ -537,8 +537,7 @@ jobs: - name: Setup uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: - # Publish runs on tags; disable the uv cache there to avoid - # release-time cache poisoning. + # Avoid release-time cache poisoning. enable-cache: ${{ !startsWith(github.ref, 'refs/tags/') }} - name: Setup Flutter @@ -607,3 +606,33 @@ jobs: publish_pkg src/serious_python_linux sleep 600 publish_pkg src/serious_python + + release: + name: Publish GitHub Release + needs: + - publish + runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.publish.result == 'success' && startsWith(github.ref, 'refs/tags/v') }} + permissions: + contents: write + steps: + - name: Detect pre-release + id: prerelease + # Dart/pub pre-releases carry a semver hyphen suffix (e.g. v1.2.0-beta.1); + # mark those as GitHub pre-releases. + env: + REF_NAME: ${{ github.ref_name }} + run: | + if [[ "$REF_NAME" == *-* ]]; then + echo "is_prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "is_prerelease=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 + with: + name: ${{ github.ref_name }} + tag_name: ${{ github.ref_name }} + prerelease: ${{ steps.prerelease.outputs.is_prerelease == 'true' }} + generate_release_notes: true diff --git a/docs/dedicated-data-channels.md b/docs/dedicated-data-channels.md index aab40476..e6c4f5df 100644 --- a/docs/dedicated-data-channels.md +++ b/docs/dedicated-data-channels.md @@ -288,7 +288,11 @@ def _pump(self): Real parallelism comes from C extensions that **release the GIL during their work** — NumPy, PyTorch, Pillow, cryptography. Pure-Python CPU work serialises on the GIL; for that, use `multiprocessing` or -PEP 684 subinterpreters. +PEP 684 subinterpreters. Caveat: `multiprocessing` works in the embedded +runtime only on **desktop** hosts whose binary services the spawn re-exec +protocol via `serious_python_main` (dart_bridge >= 1.5.0, flet >= 0.86 build +template); it is not available on iOS/Android, where the OS forbids spawning +child processes. ### Dart side — Isolate scope diff --git a/src/serious_python/CHANGELOG.md b/src/serious_python/CHANGELOG.md index dfa7f545..8e4d189a 100644 --- a/src/serious_python/CHANGELOG.md +++ b/src/serious_python/CHANGELOG.md @@ -1,3 +1,71 @@ +## 4.5.1 + +* **iOS/macOS:** the bundled `Python`, `dart_bridge` and stdlib extension XCFrameworks are now signed on both layers — each slice's inner `.framework` as well as the outer `.xcframework`. 4.5.0 signed only the outer bundle. Note this did **not** change `isSecureTimestamp`, which still reports false — that field appears not to be reachable by signing; `signed`, which `ITMS-91065` names, is true. See `serious_python_darwin` 4.5.1. +* Packaging now verifies both layers, so an unsigned slice fails the build instead of surfacing in an App Store submission. Set `SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES=require` for release builds. +* Bundled python-build snapshot re-pinned to **20260730** (`dart_bridge` **1.7.0 → 1.7.1**). No Python version moved — **3.12.13 / 3.13.14 / 3.14.6** and Pyodide are unchanged from 20260729. + +## 4.5.0 + +* **iOS/macOS:** the pre-built `Python`, `dart_bridge` and stdlib extension XCFrameworks are now staged byte-for-byte and never modified. `SERIOUS_PYTHON_BUNDLE_ID` now namespaces only the frameworks built from *your* app's wheels; the pre-built ones keep the stable `dev.flet.*` identifiers their publisher assigned. +* **This corrects 4.4.2.** Rewriting those identifiers was a hypothesis for the `ITMS-91065: Missing signature` App Store rejection (flet-dev/flet#6724) and it was wrong — the rejection reports on the publisher's signature of each XCFramework, which Xcode records in the IPA separately from your app's own signature, and the identifier rewrite destroyed it. See `serious_python_darwin` for the full explanation. +* **iOS:** use the **Swift Package Manager** packaging path for App Store submissions. The CocoaPods path copies inner `.framework` bundles and cannot produce the SDK-origin signature receipts Apple's scan looks for; it now warns during the build. +* New `SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES` (`warn` default / `require` / `off`) and `SERIOUS_PYTHON_EXPECTED_TEAM_ID` env vars verify those publisher signatures during packaging. Set `require` for release builds. +* Bundled python-build snapshot re-pinned to **20260729** (`dart_bridge` **1.6.1 → 1.7.0**). No Python version moved — **3.12.13 / 3.13.14 / 3.14.6** and Pyodide are unchanged from 20260727 — but every Apple XCFramework in that release is now signed by the Flet publishing team with a secure timestamp. See `serious_python_darwin` 4.5.0. + +## 4.4.2 + +* **iOS:** new `SERIOUS_PYTHON_BUNDLE_ID` env var namespaces the generated frameworks' bundle identifiers under your app (`com.example.myapp.-ssl`) instead of a shared `org.python.*` default identical in every app built with serious_python. `flet build` sets it for you; set it yourself for a manual two-step build, in both places you set `SERIOUS_PYTHON_SITE_PACKAGES`. +* **iOS:** the bundled Python frameworks (`_ssl`, `_hashlib` and every other native extension) now carry the build-provenance keys Xcode stamps into a real framework's `Info.plist`, instead of a minimal hand-written one. +* Both are hypotheses under test for the `ITMS-91065: Missing signature` App Store rejection in flet-dev/flet#6724 — neither is a confirmed fix. Only new App Store submissions are affected by that rejection; existing published apps are not. See `serious_python_darwin` 4.4.2. +* Bundled python-build snapshot re-pinned to **20260727**. No versions moved — Python (**3.12.13 / 3.13.14 / 3.14.6**), Pyodide, and `dart_bridge` **1.6.1** are unchanged from 20260726. + +## 4.4.1 + +* **iOS:** the bundled `_ssl` and `_hashlib` frameworks now ship OpenSSL's official privacy manifest instead of an incomplete stub, correcting what they declare under Apple's third-party SDK requirements. This is a correctness fix and is *not* expected to resolve the `ITMS-91065: Missing signature` App Store rejection reported in flet-dev/flet#6724, which remains under investigation. See `serious_python_darwin` 4.4.1. +* Bundled python-build snapshot re-pinned to **20260726**. No versions moved — Python (**3.12.13 / 3.13.14 / 3.14.6**), Pyodide, and `dart_bridge` **1.6.1** are unchanged from 20260725. + +## 4.4.0 + +* **iOS/macOS:** fix built iOS apps crashing at startup with `Failed to lookup symbol 'serious_python_run'`. `dart_bridge` now ships as a dynamic framework, so the FFI entry points Dart and Python resolve via `dlsym` stay exported; previously it was static-linked into the app executable, which exports nothing. Release/device builds only. See `serious_python_darwin` 4.4.0. +* Bundled python-build snapshot re-pinned to **20260725** (`dart_bridge` **1.5.1 → 1.6.1**, which carries the dynamic-framework change above; Pyodide 3.14 **314.0.2 → 314.0.3**). Python versions (**3.12.13 / 3.13.14 / 3.14.6**) are unchanged. + +## 4.3.6 + +* **Android:** PEP 734 subinterpreters (Python 3.14's `concurrent.interpreters` / `InterpreterPoolExecutor`) now work in built apps. Previously the main interpreter could import them, but every *subinterpreter* failed to import any relocated C extension (`ModuleNotFoundError: _struct` / `_interpqueues` / ...) — which broke the whole feature (its cross-interpreter transport pickles → `_struct`, and its queues need `_interpqueues`). The native-module finder lives on `sys.meta_path`, which is per-interpreter, and was installed only in the main interpreter. See `serious_python_android` 4.3.6. +* **Windows:** fix startup with non-ASCII app paths or environment values — Dart FFI strings (UTF-8) are now converted to UTF-16 before being passed to the Windows CRT, so paths/env values are no longer corrupted through the process ANSI code page, and Python UTF-8 mode is enabled before `Py_Initialize()`. See `serious_python_windows` 4.3.6 and flet-dev/flet#6641. +* Bundled python-build snapshot re-pinned to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**, which carries the Windows fix above). Python versions (**3.12.13 / 3.13.14 / 3.14.6**) are unchanged. + +## 4.3.4 + +* **Windows/Linux:** desktop apps on Python **3.14** no longer crash at startup with `ModuleNotFoundError: No module named '_pyrepl'` when the app (or a dependency, e.g. NLTK) imports `pydoc` or `pdb`. `_pyrepl` was pruned from the desktop stdlib as a dev-only module, but 3.14's `pydoc`/`pdb` import it at module load. See `serious_python_windows` / `serious_python_linux` 4.3.4 and flet-dev/serious-python#236. +* Bundled python-build snapshot re-pinned to **20260719**; the only change vs 20260714 is the desktop `_pyrepl` un-prune above. All runtime versions (Python **3.12.13 / 3.13.14 / 3.14.6**, `dart_bridge` **1.5.0**) are unchanged from 4.3.3. +* **Android:** fix code edits not taking effect under `flet debug android` — the app kept running stale code after a re-run because the on-device extraction cache wasn't invalidated by a same-version reinstall. See `serious_python_android` 4.3.4 and flet-dev/flet#6682. + +## 4.3.3 + +* **Windows:** fix `flet build windows` failing on non-UTF-8 system locales (e.g. Simplified-Chinese Windows, code page **936/GBK**) with `error C2220` (escalated from `warning C4819`) while compiling the Windows plugin — a non-ASCII character in a source comment couldn't be decoded under GBK and the Flutter template's `/WX` made it fatal. The character is removed and the plugin now builds with `/utf-8`. See `serious_python_windows` 4.3.3 and flet-dev/flet#6686. +* No runtime changes: bundled Python versions (**3.12.13 / 3.13.14 / 3.14.6**) and `dart_bridge` (**1.5.0**) are unchanged from 4.3.2. + +## 4.3.2 + +* **iOS:** interdependent bundled dylibs (e.g. pyarrow, llama-cpp-python) no longer crash the app at launch with `dyld: Library not loaded: @rpath/lib.dylib` — the framework install-ids and sibling `@rpath` references are reconciled to the relocated framework paths ([#223](https://github.com/flet-dev/serious-python/issues/223)). See `serious_python_darwin` 4.3.2. +* **Android:** fix `import ` yielding an empty module for a package whose `__init__` is itself the native extension (e.g. apsw) — the extension is now resolved from its `/__init__.soref` marker. See `serious_python_android` 4.3.2. +* Bump the bundled python-build snapshot to `20260714`, fixing three on-device crashes. Bundled Python versions (**3.12.13 / 3.13.14 / 3.14.6**) and `dart_bridge` (**1.5.0**) are unchanged from 4.3.1. +* **Android:** fix 3.13/3.14 apps crashing with `SIGSYS` before any app code runs on x86_64 (and other ABIs with a 32-bit-style `SYS_open`) — mimalloc's raw `open(2)` syscall is forbidden by Android's seccomp policy. See `serious_python_android` 4.3.2. +* **iOS/Android:** `_pyrepl` is no longer pruned from the mobile stdlib — Python 3.14's `pdb` imports it at module load, so anything importing `pdb` (e.g. pytest's debugging plugin) failed with `ModuleNotFoundError: No module named '_pyrepl'`. See `serious_python_darwin` / `serious_python_android` 4.3.2. +* **iOS:** `import multiprocessing` (e.g. via scikit-learn/joblib) no longer fails with `ModuleNotFoundError: No module named '_posixshmem'` — the `_posixshmem` extension is now built into the iOS runtimes alongside `_multiprocessing`. See `serious_python_darwin` 4.3.2. + +## 4.3.1 + +* **Android:** fix `flet build apk --arch x86_64` (or any `--arch` subset not including `arm64-v8a`) producing an APK with an **empty `sitepackages.zip`** — the app shipped without its Python dependencies and the first import failed at startup. See `serious_python_android` 4.3.1. + +## 4.3.0 + +* **Desktop multiprocessing support** ([flet-dev/flet#4283](https://github.com/flet-dev/flet/issues/4283)). `dart_bridge` **1.5.0** adds `serious_python_is_mp_invocation` / `serious_python_main` (+ `_w` wide-char variants on Windows): host apps call them first thing in `main` to detect CPython child command lines (`--multiprocessing-fork`, `-c "from multiprocessing..."` — spawn workers, the resource tracker, and the forkserver) and service them as a plain headless interpreter (`Py_Main`/`Py_BytesMain`, stable ABI) instead of re-launching the GUI. The exports rely on the `PYTHONHOME`/`PYTHONPATH` the parent already stamped process-wide. +* `PYTHONINSPECT=1` is no longer set by any platform implementation. It had no effect on the embedded interpreter, but it leaked into the process environment where any *real* interpreter child (e.g. a serviced multiprocessing worker) would inherit it and hang in interactive mode after its command completed. +* Bump the bundled python-build snapshot to `20260708`, which delivers `dart_bridge` **1.5.0**; Pyodide for 3.14 bumped **314.0.1 → 314.0.2**. Bundled Python versions are unchanged from 4.2.1 (**3.12.13 / 3.13.14 / 3.14.6**). +* **Windows:** fix `flet build windows` failing with `file INSTALL cannot find "C:/WINDOWS/System32/vcruntime140_1.dll"` for users who build with VS Build Tools rather than full Visual Studio (a WOW64 file-system-redirection issue with the bundled 32-bit cmake). See `serious_python_windows` 4.3.0. + ## 4.2.1 * **iOS/macOS:** ctypes packages that ship plain `.dylib` shared libraries (e.g. `llama-cpp-python`'s `libllama` / `libggml`) now load on the **iOS simulator**. Such `.dylib`s are now packaged as per-slice xcframeworks (previously only `.so` C-extensions were), so they carry a simulator slice instead of shipping the device build and failing `dlopen` with `incompatible platform (have 'iOS', need 'iOS-simulator')`; their install-name is preserved so multi-lib packages still resolve their sibling libs. See `serious_python_darwin` 4.2.1. diff --git a/src/serious_python/README.md b/src/serious_python/README.md index 80a0ad27..8f73160e 100644 --- a/src/serious_python/README.md +++ b/src/serious_python/README.md @@ -192,6 +192,28 @@ dart run serious_python:main package app/src -p iOS -r -r -r app/src/requirement For the **web** (`Emscripten`) target there is no `SERIOUS_PYTHON_APP`; the app and its `__pypackages__` are zipped into the `app/app.zip` asset instead — make sure it's added to `pubspec.yaml`. +On **iOS**, also set `SERIOUS_PYTHON_BUNDLE_ID` to your app's bundle identifier: + +``` +export SERIOUS_PYTHON_BUNDLE_ID=com.example.myapp +``` + +Each native extension in your app's own dependencies ships as its own embedded +framework, and this namespaces their `CFBundleIdentifier`s under your app +(`com.example.myapp.-numpy-core-multiarray`) instead of a shared `org.python.*` +default — matching what CPython's own iOS support does. Set it in **both** places +`SERIOUS_PYTHON_SITE_PACKAGES` is set (the `package` command and the later `flutter +build`), since the CocoaPods `prepare_command` re-runs the darwin sync. If unset — +or if the value isn't a valid bundle identifier — the `org.python.*` defaults are +kept and a warning is printed. `flet build` sets this for you. + +This applies **only** to frameworks built here from your wheels. `Python.xcframework`, +`dart_bridge.xcframework`, and the stdlib extension frameworks arrive pre-built and +**signed by their publisher**, with stable `dev.flet.python.*` / `dev.flet.dartbridge` +identifiers assigned upstream. serious_python stages those byte-for-byte and never +rewrites anything inside them — see [SDK-origin signatures](#sdk-origin-signatures) +below. + ## Python app structure By default, embedded Python program is run in a separate thread, to avoid UI blocking. Your Flutter app is not supposed to directly call Python functions or modules, but instead it should communicate via some API provided by a Python app, such as: REST API, sockets, SQLite database, files, etc. @@ -249,6 +271,54 @@ The on-disk layout differs per platform, mostly because each OS has different ru The CPython runtime, stdlib, and (on iOS) native extensions are bundled into `serious_python_darwin.framework` as resources. On **iOS**, the App Store forbids loose `.dylib`s, so every native extension `.so` is repackaged into a signed `.framework` inside an `.xcframework`, with a `.fwork` text marker left at the module's import path; CPython's `AppleFrameworkLoader` reads the marker and loads the framework binary. On **macOS**, native extensions stay as plain `.so`, merged into universal (`arm64`+`x86_64`) binaries at package time. `PYTHONHOME` is the framework's resource path; `sys.path` includes `/site-packages`, `/stdlib`, and `/stdlib/lib-dynload`. +#### SDK-origin signatures + +Xcode records, for every `.xcframework` your app links against, whether **the +publisher** signed it and whether that signature carried a secure timestamp. The +result is written into the IPA as `Signatures/.xcframework-ios.signature`, +and Apple's App Store scan reports a missing one as `ITMS-91065: Missing signature`. + +This is **separate from your app's own signature.** Xcode re-signs every embedded +framework with your Apple Distribution identity at embed time and again at +`exportArchive`; that has no effect on the SDK-origin receipt. Equally, editing a +single file inside an `.xcframework` — even one `Info.plist` key — invalidates the +publisher's signature and turns the receipt back into `signed = false`. + +Both layers are signed: each slice's inner `.framework` as well as the outer +`.xcframework`, in that order. Signing only the outer bundle yields a receipt +reading `signed = true` but `isSecureTimestamp = false`. + +So serious_python treats `Python.xcframework`, `dart_bridge.xcframework`, and the +stdlib extension frameworks as **immutable** once downloaded. They are copied +verbatim at every staging step, and each step re-checks a digest manifest recorded +at extraction time, so a reintroduced mutation fails the build instead of surfacing +as an App Store rejection weeks later. + +`SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES` controls what happens when a publisher +signature is missing or invalid: + +| value | behaviour | +| :--- | :--- | +| `warn` (default) | report and continue — lets you pin an older, pre-signing `python-build` / `dart-bridge` release | +| `require` | fail the build; use this for App Store submissions | +| `off` | skip the signature checks | + +Set `SERIOUS_PYTHON_EXPECTED_TEAM_ID` to additionally require a specific Apple +Team ID on those signatures. The digest-manifest checks always run and are always +fatal — they cover this package's own behaviour, not the artifacts you pinned. + +#### iOS packaging path: use SwiftPM + +The **Swift Package Manager** path declares the publisher's `.xcframework`s as +`binaryTarget`s, which is what makes Xcode emit the `Signatures/` receipts above. + +The CocoaPods path copies the *inner* `.framework` bundles out of the stdlib +xcframeworks in a `Pods-Runner-frameworks.sh` script phase, which discards the +outer-XCFramework provenance even when the source is correctly signed. It still +produces a working app, but it cannot produce complete SDK-origin receipts. **Use +the SwiftPM path for App Store submissions** until that path is replaced with real +vendored XCFramework declarations. + ### Linux / Windows specifics The CPython runtime (`libpython3.so` + `libpython.so` on Linux; `python3.dll` + `python.dll` on Windows), `libdart_bridge`, the stdlib, and native modules are copied next to your app's executable at build time. `PYTHONHOME` is the executable's directory. On Windows, extension modules (`.pyd`) and their dependent DLLs live in `/DLLs/`, which is added to `sys.path`. diff --git a/src/serious_python/example/bridge_example/pubspec.yaml b/src/serious_python/example/bridge_example/pubspec.yaml index 23133af7..fd4e23bb 100644 --- a/src/serious_python/example/bridge_example/pubspec.yaml +++ b/src/serious_python/example/bridge_example/pubspec.yaml @@ -24,5 +24,14 @@ dev_dependencies: sdk: flutter flutter_lints: ^2.0.0 +# jni arrives transitively (path_provider -> path_provider_android -> +# jni_flutter -> jni). 1.0.1 added global_jni_env.c, which passes a `va_list` +# where a `void *` is expected: fine on x86_64, where va_list is an array type +# that decays to a pointer, but a hard error on aarch64, where it's a struct. +# That breaks `flutter build linux` on ARM64 hosts. Tracked upstream at +# dart-lang/native#3498 — drop this override once a fixed jni is released. +dependency_overrides: + jni: 1.0.0 + flutter: uses-material-design: true diff --git a/src/serious_python/example/flask_example/pubspec.lock b/src/serious_python/example/flask_example/pubspec.lock index a0c8b512..921d732a 100644 --- a/src/serious_python/example/flask_example/pubspec.lock +++ b/src/serious_python/example/flask_example/pubspec.lock @@ -156,7 +156,7 @@ packages: source: hosted version: "4.1.2" jni: - dependency: transitive + dependency: "direct overridden" description: name: jni sha256: c2230682d5bc2362c1c9e8d3c7f406d9cbba23ab3f2e203a025dd47e0fb2e68f @@ -361,42 +361,42 @@ packages: path: "../.." relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_android: dependency: transitive description: path: "../../../serious_python_android" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_darwin: dependency: transitive description: path: "../../../serious_python_darwin" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_linux: dependency: transitive description: path: "../../../serious_python_linux" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_platform_interface: dependency: transitive description: path: "../../../serious_python_platform_interface" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_windows: dependency: transitive description: path: "../../../serious_python_windows" relative: true source: path - version: "4.0.0" + version: "4.4.1" shelf: dependency: transitive description: diff --git a/src/serious_python/example/flask_example/pubspec.yaml b/src/serious_python/example/flask_example/pubspec.yaml index 47da6963..23df2097 100644 --- a/src/serious_python/example/flask_example/pubspec.yaml +++ b/src/serious_python/example/flask_example/pubspec.yaml @@ -56,6 +56,15 @@ dev_dependencies: # following page: https://dart.dev/tools/pub/pubspec # The following section is specific to Flutter packages. +# jni arrives transitively (path_provider -> path_provider_android -> +# jni_flutter -> jni). 1.0.1 added global_jni_env.c, which passes a `va_list` +# where a `void *` is expected: fine on x86_64, where va_list is an array type +# that decays to a pointer, but a hard error on aarch64, where it's a struct. +# That breaks `flutter build linux` on ARM64 hosts. Tracked upstream at +# dart-lang/native#3498 — drop this override once a fixed jni is released. +dependency_overrides: + jni: 1.0.0 + flutter: # The following line ensures that the Material Icons font is diff --git a/src/serious_python/example/run_example/pubspec.lock b/src/serious_python/example/run_example/pubspec.lock index 6d374be4..21ba23f7 100644 --- a/src/serious_python/example/run_example/pubspec.lock +++ b/src/serious_python/example/run_example/pubspec.lock @@ -171,7 +171,7 @@ packages: source: sdk version: "0.0.0" jni: - dependency: transitive + dependency: "direct overridden" description: name: jni sha256: c2230682d5bc2362c1c9e8d3c7f406d9cbba23ab3f2e203a025dd47e0fb2e68f @@ -384,42 +384,42 @@ packages: path: "../.." relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_android: dependency: transitive description: path: "../../../serious_python_android" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_darwin: dependency: transitive description: path: "../../../serious_python_darwin" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_linux: dependency: transitive description: path: "../../../serious_python_linux" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_platform_interface: dependency: transitive description: path: "../../../serious_python_platform_interface" relative: true source: path - version: "4.0.0" + version: "4.4.1" serious_python_windows: dependency: transitive description: path: "../../../serious_python_windows" relative: true source: path - version: "4.0.0" + version: "4.4.1" shelf: dependency: transitive description: diff --git a/src/serious_python/example/run_example/pubspec.yaml b/src/serious_python/example/run_example/pubspec.yaml index 3035fb7e..5e3c3c30 100644 --- a/src/serious_python/example/run_example/pubspec.yaml +++ b/src/serious_python/example/run_example/pubspec.yaml @@ -58,6 +58,15 @@ dev_dependencies: # following page: https://dart.dev/tools/pub/pubspec # The following section is specific to Flutter packages. +# jni arrives transitively (path_provider -> path_provider_android -> +# jni_flutter -> jni). 1.0.1 added global_jni_env.c, which passes a `va_list` +# where a `void *` is expected: fine on x86_64, where va_list is an array type +# that decays to a pointer, but a hard error on aarch64, where it's a struct. +# That breaks `flutter build linux` on ARM64 hosts. Tracked upstream at +# dart-lang/native#3498 — drop this override once a fixed jni is released. +dependency_overrides: + jni: 1.0.0 + flutter: # The following line ensures that the Material Icons font is diff --git a/src/serious_python/lib/src/python_versions.dart b/src/serious_python/lib/src/python_versions.dart index d654a80b..f79a851b 100644 --- a/src/serious_python/lib/src/python_versions.dart +++ b/src/serious_python/lib/src/python_versions.dart @@ -1,5 +1,5 @@ // GENERATED by `dart run serious_python:gen_version_tables` from python-build's -// manifest.json (release 20260701). Do not edit by hand — edit python-build's +// manifest.json (release 20260730). Do not edit by hand — edit python-build's // manifest.json, cut a release, bump `pythonReleaseDate`, and regenerate. const pythonVersionEnvironmentVariable = "SERIOUS_PYTHON_VERSION"; @@ -10,8 +10,8 @@ const pyodideVersionEnvironmentVariable = "SERIOUS_PYTHON_PYODIDE_VERSION"; const dartBridgeVersionEnvironmentVariable = "DART_BRIDGE_VERSION"; /// python-build release the bundled runtimes come from (YYYYMMDD). -const pythonReleaseDate = "20260701"; -const dartBridgeVersion = "1.4.1"; +const pythonReleaseDate = "20260730"; +const dartBridgeVersion = "1.7.1"; const defaultPythonVersion = "3.14"; class PythonRelease { @@ -56,7 +56,7 @@ const pythonReleases = { "3.14": PythonRelease( standaloneVersion: "3.14.6", standaloneReleaseDate: "20260623", - pyodideVersion: "314.0.1", + pyodideVersion: "314.0.3", pyodidePlatformTag: "pyemscripten-2026.0-wasm32", androidAbis: ["arm64-v8a", "x86_64", "armeabi-v7a"], prerelease: false, diff --git a/src/serious_python/pubspec.yaml b/src/serious_python/pubspec.yaml index 97d1a686..f31870f6 100644 --- a/src/serious_python/pubspec.yaml +++ b/src/serious_python/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python description: A cross-platform plugin for adding embedded Python runtime to your Flutter apps. homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 platforms: ios: diff --git a/src/serious_python_android/CHANGELOG.md b/src/serious_python_android/CHANGELOG.md index 369c0781..5fe559c3 100644 --- a/src/serious_python_android/CHANGELOG.md +++ b/src/serious_python_android/CHANGELOG.md @@ -1,3 +1,53 @@ +## 4.5.1 + +* Re-pins the bundled python-build snapshot to **20260730** and `dart_bridge` to **1.7.1**. Both exist to sign the Apple XCFrameworks on both layers (see `serious_python_darwin` 4.5.1); no Python version moved from 20260729 (**3.12.13 / 3.13.14 / 3.14.6**), and nothing about Android changed. + +## 4.5.0 + +* Re-pins the bundled python-build snapshot to **20260729** and `dart_bridge` to **1.7.0**. Both exist to provider-sign the Apple XCFrameworks (see `serious_python_darwin` 4.5.0). Nothing about Android changed: no Python version moved from 20260727 (**3.12.13 / 3.13.14 / 3.14.6**), and `dart_bridge`'s Android `.so`s are byte-identical to 1.6.1 — verified, not assumed. + +## 4.4.2 + +* Re-pins the bundled python-build snapshot to **20260727**. Its only functional change is to the iOS frameworks' `Info.plist` (see `serious_python_darwin` 4.4.2); no versions moved and the Android runtime is unchanged from 20260726. + +## 4.4.1 + +* Re-pins the bundled python-build snapshot to **20260726**. Its only functional change is an iOS privacy-manifest fix (see `serious_python_darwin` 4.4.1); no versions moved and the Android runtime is unchanged from 20260725. + +## 4.4.0 + +* Re-pins the bundled python-build snapshot to **20260725** (`dart_bridge` **1.5.1 → 1.6.1**). 1.6.1 changes only how `dart_bridge` is packaged for Apple platforms (static library → dynamic framework, see `serious_python_darwin` 4.4.0); the Android runtime is byte-identical to 20260720. + +## 4.3.6 + +* **PEP 734 subinterpreters now work** (Python 3.14 `concurrent.interpreters` / `InterpreterPoolExecutor`) — enabling true multi-core CPU parallelism inside a single process, which `multiprocessing` cannot provide on Android. Previously the main interpreter imported the machinery fine, but every subinterpreter raised `ModuleNotFoundError` for `_struct` / `_interpqueues` / any other C extension, so `InterpreterPoolExecutor` and the low-level API were unusable. Cause: Android relocates C extensions (native-mmap packaging) and resolves them through `_SorefFinder` on `sys.meta_path`; `sys.meta_path` is per-interpreter, and `install()` only ran in the main interpreter, so a freshly created subinterpreter had a `meta_path` without the finder. Fix: `_sp_bootstrap.install()` now also wraps `concurrent.interpreters.create()` so every new interpreter installs the finder before use. The install runs via `Interpreter.exec()` — a source string, which is pickle-free (unlike `Interpreter.call()`), so it works *before* `_struct` is importable in the child. Because `InterpreterPoolExecutor` builds its workers via `concurrent.interpreters.create()`, the pool is fixed transparently with no user-code change. A no-op before 3.14; iOS/desktop were unaffected. +* Re-pins the bundled python-build snapshot to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**). 1.5.1 is a Windows-only UTF-8 startup fix (see `serious_python_windows` 4.3.6); the Android runtime is functionally unchanged from 20260719. + +## 4.3.4 + +* Re-pins the bundled python-build snapshot to **20260719** (previously 20260714). The snapshot un-prunes `_pyrepl` on Windows/Linux desktop only; Android already shipped `_pyrepl` (un-pruned in 4.3.2), so these runtimes are byte-identical to 20260714. +* **Android:** fix code edits not taking effect under `flet debug android` — the app kept running the previously-unpacked, stale code after a re-run ([flet-dev/flet#6682](https://github.com/flet-dev/flet/issues/6682)). `prepareApp` copies the app payload out of the APK only when its cache key changes, and the key was `versionName+versionCode`. Since `flet debug` reinstalls the same-version APK on each iteration (`flutter run` does an update install that preserves app data, including the cache marker), the key never changed and re-extraction was skipped. The key now also includes `PackageManager.lastUpdateTime`, which is bumped on every (re)install but stays stable across plain relaunches — so a debug reinstall re-extracts the new code while ordinary relaunches still hit the cache. `flet build apk` was unaffected (fresh/version-bumped install). + +## 4.3.3 + +* Version bump aligning with the `serious_python_*` 4.3.3 release (a Windows build fix). No Android-affecting changes. + +## 4.3.2 + +* **Resolve a package whose `__init__` is itself the native extension.** `_SorefFinder` only probed `.soref`, so a package that ships its extension as `/__init__..so` — e.g. apsw (import name `apsw`), whose relocation marker lands at `apsw/__init__.soref` — was never resolved: `find_spec` returned `None`, the synthesized empty `apsw/__init__.py` won, and `import apsw` yielded an empty module (`AttributeError: module 'apsw' has no attribute 'Connection'`). `find_spec` now falls back to `/__init__.soref`, loads the extension under the correct top-level name via `ExtensionFileLoader`, and marks the result a package (with `submodule_search_locations`) so pure-Python submodules (`apsw.ext`, …) still resolve. +* Bump the bundled python-build snapshot to `20260714`, fixing two on-device crashes on Python **3.13/3.14** (3.12 is unaffected; Python/`dart_bridge` versions are unchanged): + * Apps died with `SIGSYS` at `dlopen()` of `libpython` — before the interpreter even started — on **x86_64/x86/armeabi-v7a**: mimalloc (bundled with CPython since 3.13) reads `/proc/sys/vm/overcommit_memory` during allocator init via a bare `open(2)` syscall, which Android's bionic seccomp policy forbids (only `openat(2)` is allowed). python-build now patches the call to `SYS_openat(AT_FDCWD, …)`. `arm64-v8a` was latently unaffected (no `SYS_open` there, so mimalloc already went through libc `open()` → `openat`), but emulators are typically x86_64. + * `_pyrepl` is no longer pruned from the bundled stdlib: Python 3.14's `pdb` imports it at module load, so anything importing `pdb` (e.g. pytest's debugging plugin) died with `ModuleNotFoundError: No module named '_pyrepl'`. + +## 4.3.1 + +* Fix `flet build apk --arch ` shipping an **empty `sitepackages.zip`** whenever the selected ABI subset didn't include `arm64-v8a` (e.g. `--arch x86_64`) — the app bundled no Python site-packages at all and the very first dependency import failed at startup. The ABI-common pure-code zips (`sitepackages.zip` / `extract.zip`) were built from a hardcoded primary ABI (`abis.first()`, i.e. `arm64-v8a`); when only other ABIs were staged under `SERIOUS_PYTHON_SITE_PACKAGES`, the primary split task walked a nonexistent directory and silently produced a valid-but-empty zip. The primary ABI is now the first manifest ABI whose site-packages tree was actually staged. If none is staged at all (legitimate when packaging with no requirements), the build falls back to `abis.first()` and logs `sitepackages.zip will be empty` instead of staying silent. + +## 4.3.0 + +* `PYTHONINSPECT=1` is no longer set by any platform implementation. It had no effect on the embedded interpreter, but it leaked into the process environment where any *real* interpreter child (e.g. a serviced multiprocessing worker) would inherit it and hang in interactive mode after its command completed. No functional change on Android, which doesn't support process spawning. +* Bump the bundled python-build snapshot to `20260708` (`dart_bridge` **1.5.0**). The new multiprocessing child-interception exports are present in the runtime but not wired up on Android, which doesn't support process spawning. + ## 4.2.1 * Bump the bundled python-build snapshot to `20260701`; aligns with the `serious_python_*` 4.2.1 release. The Android runtimes are byte-identical to `20260630` (the release only rebuilds the iOS runtime). diff --git a/src/serious_python_android/android/build.gradle.kts b/src/serious_python_android/android/build.gradle.kts index 68d5c8a4..dc9027c1 100644 --- a/src/serious_python_android/android/build.gradle.kts +++ b/src/serious_python_android/android/build.gradle.kts @@ -21,7 +21,7 @@ buildscript { } group = "com.flet.serious_python_android" -version = "4.2.1" +version = "4.5.1" rootProject.allprojects { repositories { @@ -136,7 +136,18 @@ val extractGlobs: List = extractPackages.filter { '*' in it || '?' in it }.map(::globToRegex) val extractPlain: List = extractPackages.filter { '*' !in it && '?' !in it } -val primaryAbi = abis.first() // pure zips are ABI-common: build once +// Pure zips are ABI-common: build once, from the first ABI whose site-packages +// tree was actually staged — `flet build --arch` may stage a subset of the ABIs +// (e.g. only x86_64), and a hardcoded abis.first() would then walk a missing +// dir and silently ship an EMPTY sitepackages.zip. No staged dir at all is +// legitimate (packaged with no requirements): fall back to abis.first(), whose +// empty walk correctly yields empty zips. +val primaryAbi = abis.firstOrNull { siteSrcDir != null && File(siteSrcDir, it).isDirectory } + ?: abis.first().also { + logger.lifecycle( + "serious_python: no staged site-packages under $siteSrcDir; " + + "sitepackages.zip will be empty") + } val assetsDir = file("src/main/assets") val bootstrapPy = file("../python/_sp_bootstrap.py") diff --git a/src/serious_python_android/android/python_versions.properties b/src/serious_python_android/android/python_versions.properties index faf98a2e..0bdc3bba 100644 --- a/src/serious_python_android/android/python_versions.properties +++ b/src/serious_python_android/android/python_versions.properties @@ -1,8 +1,8 @@ # GENERATED by `dart run serious_python:gen_version_tables` from -# python-build manifest.json (release 20260701). Do not edit by hand. +# python-build manifest.json (release 20260730). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.4.1 -python_build_release_date=20260701 +dart_bridge_version=1.7.1 +python_build_release_date=20260730 3.12.full_version=3.12.13 3.12.android_abis=arm64-v8a,x86_64,armeabi-v7a 3.13.full_version=3.13.14 diff --git a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java index 238401af..41354d54 100644 --- a/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java +++ b/src/serious_python_android/android/src/main/java/com/flet/serious_python_android/AndroidPlugin.java @@ -132,7 +132,16 @@ public void onMethodCall(@NonNull MethodCall call, @NonNull Result result) { long versionCode = (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.P) ? info.getLongVersionCode() : (long) info.versionCode; - result.success(versionName + "+" + versionCode); + // Append lastUpdateTime so the value changes on every (re)install, not + // just on a version bump. `flet debug android` reinstalls the same + // versionName+versionCode APK on each iteration (`flutter run` does an + // update install that preserves app data), so without this the + // extraction cache key in prepareApp never changes and the app keeps + // running the previously-unpacked, stale code. PackageManager bumps + // lastUpdateTime on each install while leaving it stable across plain + // relaunches, so the cache is still hit when nothing was reinstalled. + long lastUpdateTime = info.lastUpdateTime; + result.success(versionName + "+" + versionCode + "+" + lastUpdateTime); } catch (Exception e) { result.error("Error", e.getMessage(), null); } diff --git a/src/serious_python_android/lib/serious_python_android.dart b/src/serious_python_android/lib/serious_python_android.dart index 3b3f779b..25845459 100644 --- a/src/serious_python_android/lib/serious_python_android.dart +++ b/src/serious_python_android/lib/serious_python_android.dart @@ -45,6 +45,11 @@ class SeriousPythonAndroid extends SeriousPythonPlatform { final siteZip = p.join(base, 'sitepackages.zip'); final extractDir = p.join(base, 'extract'); + // `getAppVersion` returns `versionName+versionCode+lastUpdateTime`, so the + // key changes on every (re)install — including `flet debug`'s repeated + // same-version `flutter run` reinstalls — while staying stable across plain + // relaunches. Keeping only versionName+versionCode here would make the + // debug workflow keep running previously-unpacked, stale app code. final appVersion = await _appVersion(); final key = appVersion != null ? 'app:$appVersion' : 'app:dev'; final marker = File(p.join(base, '.key')); @@ -116,7 +121,6 @@ class SeriousPythonAndroid extends SeriousPythonPlatform { } final env = { - 'PYTHONINSPECT': '1', 'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONNOUSERSITE': '1', 'PYTHONUNBUFFERED': '1', diff --git a/src/serious_python_android/pubspec.yaml b/src/serious_python_android/pubspec.yaml index 04edf59e..4a422008 100644 --- a/src/serious_python_android/pubspec.yaml +++ b/src/serious_python_android/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python_android description: Android implementation of the serious_python plugin homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 environment: sdk: ">=3.0.0 <4.0.0" diff --git a/src/serious_python_android/python/_sp_bootstrap.py b/src/serious_python_android/python/_sp_bootstrap.py index be29d3c6..01a6aafa 100644 --- a/src/serious_python_android/python/_sp_bootstrap.py +++ b/src/serious_python_android/python/_sp_bootstrap.py @@ -1,23 +1,23 @@ """serious_python Android import bootstrap. -Installed by the dart-bridge embedder *before* ``site`` runs, via the fixed call:: +Installed by the dart-bridge embedder *before* `site` runs, via the fixed call:: import _sp_bootstrap; _sp_bootstrap.install() -It registers a ``sys.meta_path`` finder that resolves native CPython extension -modules which the build relocated into ``jniLibs//`` as real ``lib.so`` +It registers a `sys.meta_path` finder that resolves native CPython extension +modules which the build relocated into `jniLibs//` as real `lib.so` files (loaded by basename through the Android linker namespace, exactly like -``libdart_bridge.so``). Pure ``.py``/``.pyc`` modules are left to ``zipimport`` / -``FileFinder`` — this finder returns ``None`` for them. +`libdart_bridge.so`). Pure `.py`/`.pyc` modules are left to `zipimport` / +`FileFinder` — this finder returns `None` for them. -For every relocated extension the build leaves a ``.soref`` marker at the module's -original path; its content is the ``lib.so`` filename. The marker is read -**lazily** in ``find_spec`` via the frozen ``zipimport`` ``get_data`` API (for zip -entries) or a plain ``open`` (for entries extracted to disk, e.g. ``extract.zip``). +For every relocated extension the build leaves a `.soref` marker at the module's +original path; its content is the `lib.so` filename. The marker is read +**lazily** in `find_spec` via the frozen `zipimport` `get_data` API (for zip +entries) or a plain `open` (for entries extracted to disk, e.g. `extract.zip`). CRITICAL: this module must load and run *before any native module is resolvable*, -so it imports **only builtin/frozen** machinery — ``sys``, ``zipimport``, -``importlib.machinery`` — and never ``zipfile``/``struct``/``zlib`` (which would be +so it imports **only builtin/frozen** machinery — `sys`, `zipimport`, +`importlib.machinery` — and never `zipfile`/`struct`/`zlib` (which would be a chicken-and-egg: those are themselves native). """ @@ -48,7 +48,7 @@ def _apk_native_prefix(): class _SorefFinder: - """meta_path finder: dotted name -> jniLibs lib via its ``.soref`` marker.""" + """meta_path finder: dotted name -> jniLibs lib via its `.soref` marker.""" def __init__(self): # Cache one zipimporter per zip sys.path entry. Value is None for entries @@ -69,11 +69,14 @@ def _zipimporter(self, entry): return zi def _read_marker(self, member): - """Return the soname recorded in ``member`` (.soref), or None if absent. - - Probes every current ``sys.path`` entry: zip entries via the frozen - ``zipimport.get_data`` (known member, no native deps), directory entries - via a plain ``open`` (covers packages unpacked from ``extract.zip``). + """Return `(soref_bytes, sys.path entry)` for `member`, or + `(None, None)` if absent. + + Probes every current `sys.path` entry: zip entries via the frozen + `zipimport.get_data` (known member, no native deps), directory entries + via a plain `open` (covers packages unpacked from `extract.zip`). The + winning entry is returned too so a package whose `__init__` is the + native extension can locate its pure-Python submodules beside it. """ for entry in sys.path: if not entry: @@ -81,7 +84,7 @@ def _read_marker(self, member): zi = self._zipimporter(entry) if zi is not None: try: - return zi.get_data(member) # archive-relative member path + return zi.get_data(member), entry # archive-relative member except Exception: continue else: @@ -89,14 +92,21 @@ def _read_marker(self, member): path = entry + "/" + member try: with open(path, "rb") as f: - return f.read() + return f.read(), entry except OSError: continue - return None + return None, None def find_spec(self, fullname, path=None, target=None): - member = fullname.replace(".", "/") + _MARKER_SUFFIX - data = self._read_marker(member) + base = fullname.replace(".", "/") + # A plain extension module: its marker is ".soref". + data, entry = self._read_marker(base + _MARKER_SUFFIX) + is_package = False + if data is None: + # A package whose __init__ IS the native extension (e.g. apsw ships + # apsw/__init__..so): the marker sits at "/__init__.soref". + data, entry = self._read_marker(base + "/__init__" + _MARKER_SUFFIX) + is_package = data is not None if data is None: return None # not a relocated native module -> let others handle it soname = data.decode("utf-8").strip() @@ -115,11 +125,19 @@ def find_spec(self, fullname, path=None, target=None): if origin == soname and self._apk_prefix: origin = self._apk_prefix + soname loader = ExtensionFileLoader(fullname, origin) - return ModuleSpec(fullname, loader, origin=origin) - - -def install(): - """Insert the finder at the front of ``sys.meta_path`` (idempotent).""" + spec = ModuleSpec(fullname, loader, origin=origin) + if is_package and entry is not None: + # The native __init__ lives in jniLibs, but the package's pure-Python + # submodules (e.g. apsw.ext) sit at "//" in the winning + # sys.path entry (sitepackages.zip or an extract.zip dir). Point + # __path__ there so `import .` resolves via the normal + # zipimport/FileFinder machinery. + spec.submodule_search_locations = [entry + "/" + base] + return spec + + +def _install_finder(): + """Insert the finder at the front of `sys.meta_path` (idempotent).""" global _installed if _installed: return @@ -127,8 +145,11 @@ def install(): # loaded during interpreter core-init, BEFORE the finder existed — which only works # if it was builtin/frozen. A non-empty list means that module must be made static # (PyImport_AppendInittab) or it will fail under modern packaging. Expected: empty. - pre = [n for n, m in sys.modules.items() - if getattr(m, "__file__", None) and str(m.__file__).endswith(".so")] + pre = [ + n + for n, m in sys.modules.items() + if getattr(m, "__file__", None) and str(m.__file__).endswith(".so") + ] if pre: sys.stderr.write("SP_BOOTSTRAP pre-finder native modules: %r\n" % (pre,)) for f in sys.meta_path: @@ -137,3 +158,64 @@ def install(): return sys.meta_path.insert(0, _SorefFinder()) _installed = True + + +def _patch_subinterpreters(): + """Make PEP 734 subinterpreters (Python 3.14+) able to import relocated + native modules. + + `sys.meta_path` is *per-interpreter*: a subinterpreter created by + `concurrent.interpreters` / `InterpreterPoolExecutor` starts with a fresh + meta_path that does NOT contain the `_SorefFinder` installed by `_install_finder`, + so it can import no relocated C extension (`_struct`, `_interpqueues`, ...). Since + that machinery ships work between interpreters via pickle (`_struct`) and + cross-interpreter queues (`_interpqueues`), the whole feature is unusable. + + Wrap `concurrent.interpreters.create()` so every new interpreter installs + the finder before it is used. The install runs via `Interpreter.exec()` — a + source string, which is pickle-free (unlike `Interpreter.call()`), so it + works *before* `_struct` is importable in the child. Idempotent; a no-op + before 3.14 (module absent) and cannot fix + `InterpreterPoolExecutor(initializer=...)` because the initializer is + delivered over the same broken pickle path — so it must run here, at + creation time. + + NOTE: this relies on callers resolving `interpreters.create` as a module + attribute at call time (as `InterpreterPoolExecutor` does on 3.14) rather + than binding it via `from concurrent.interpreters import create`. If a + future CPython changes that, the patch silently stops applying to the pool + and subinterpreter imports regress — verified working on 3.14.6. + """ + try: + from concurrent import interpreters + except Exception: + return # < 3.14, or subinterpreters unavailable + if getattr(interpreters.create, "_sp_patched", False): + return + _orig_create = interpreters.create + + def create(*args, **kwargs): + interp = _orig_create(*args, **kwargs) + try: + interp.exec("import _sp_bootstrap\n_sp_bootstrap.install()\n") + except Exception as e: + # Finder not installed in the child: it will hit the original + # ModuleNotFoundError on its first relocated import. Leave a + # breadcrumb (cf. the SP_BOOTSTRAP audit in `install`) instead of + # failing `create` outright. + sys.stderr.write("SP_BOOTSTRAP subinterpreter finder install failed: %r\n" % (e,)) + return interp + + create._sp_patched = True + interpreters.create = create + + +def install(): + """Entry point called from the dart-bridge Android bootstrap. + + Installs the native-module finder in the current interpreter and — on + 3.14+ — teaches every future subinterpreter to install it too. + Safe to call in any interpreter; idempotent. + """ + _install_finder() + _patch_subinterpreters() diff --git a/src/serious_python_darwin/CHANGELOG.md b/src/serious_python_darwin/CHANGELOG.md index 27a509b2..bbe8c163 100644 --- a/src/serious_python_darwin/CHANGELOG.md +++ b/src/serious_python_darwin/CHANGELOG.md @@ -1,3 +1,68 @@ +## 4.5.1 + +* **iOS/macOS: the bundled XCFrameworks are now signed on both layers — each slice's inner `.framework` as well as the outer `.xcframework`.** 4.5.0 shipped artifacts whose outer bundle was signed but whose inner frameworks were not. An App Store IPA built against it reported `signed = true` — the 4.5.0 fix working — but `isSecureTimestamp = false` for `Python-ios`, `_ssl`, `_hashlib` and `dart_bridge`, in both a development archive and an App Store export. Every slice of an XCFramework Apple's scan demonstrably accepts ([krzyzanowskim/OpenSSL](https://github.com/krzyzanowskim/OpenSSL) 3.6.3000) carries its own Apple Distribution signature with a secure timestamp, with the outer bundle signed last; an unsigned inner framework was the only structural difference left. See flet-dev/python-build#38 and flet-dev/dart-bridge#14. +* **This did not change `isSecureTimestamp`, and that appears not to be something signing can change.** With both layers signed, the receipts still read `signed = true` / `isSecureTimestamp = false`. The receipt's `cdhashes` entry matches the **outer** xcframework's CDHash exactly — so Xcode reads the outer signature, which carries a genuine Apple TSA `Timestamp=`, and reports `isSecureTimestamp = false` anyway. Every receipt in the archive reports false, and the receipts record `signatureType = AppleDeveloperProgram`; the accepted OpenSSL artifact is signed by an Apple Distribution identity too, so it would carry the same type. Decoding the CMS blob in `_CodeSignature/CodeSignature` settles it: our signature carries a genuine RFC 3161 `id-smime-aa-timeStampToken` (1.2.840.113549.1.9.16.2.14) unsigned attribute — a real TSA token, not just a self-asserted `signingTime` — and its attribute set is structurally identical to the accepted OpenSSL artifact's, down to the Apple certificate extensions. There is no observable difference left between the two signatures, so `isSecureTimestamp = false` cannot be caused by anything in ours. The reasonable reading is that `ITMS-91065: Missing signature` tests `signed`, which is now true, and that `isSecureTimestamp` is not reachable for a Developer Program identity. Signing both layers is kept regardless: it matches what an accepted artifact does, and it is correct practice for a distributed binary framework. +* **Consumer verification now covers both layers.** `spv_verify_provider` checks each slice's inner framework as well as the outer bundle, so an unsigned slice fails during packaging rather than surfacing in an IPA weeks later. Slice frameworks are located by glob rather than by the xcframework's name, since `stage_spm.sh` stages `Python.xcframework` as `Python-.xcframework` and a name-keyed lookup would silently find nothing there. Signature presence is probed with `codesign -dv` rather than by looking for `_CodeSignature`, because a versioned macOS bundle keeps it under `Versions//` and that name is not always `A` — CPython uses `Versions/3.14`. +* Re-pins the bundled python-build snapshot to **20260730** (`dart_bridge` **1.7.0 → 1.7.1**). No Python version moved — **3.12.13 / 3.13.14 / 3.14.6** and Pyodide are unchanged from 20260729. Verified on the published artifacts: 56 XCFrameworks and 112 slice frameworks in `python-ios-dart-3.14.6`, all signed and securely timestamped by `Apple Distribution: Appveyor Systems Inc. (GXXRQJK434)`. +* python-build 20260730 also fixes the macOS `Python.framework` layout: `Headers` and `Modules` were real directories at the framework root, which codesign rejects for a versioned bundle (`unsealed contents present in the root directory of an embedded framework`). They are now symlinks into `Versions/Current`, as CPython's own framework does for `Headers` and `Resources`. See flet-dev/python-build#39. + +## 4.5.0 + +* **iOS/macOS: provider-built XCFrameworks are now staged byte-for-byte, and their SDK-origin signatures are verified.** `Python.xcframework`, `dart_bridge.xcframework`, and every stdlib extension framework arrive pre-built from flet-dev/python-build and flet-dev/dart-bridge. serious_python no longer rewrites anything inside them: the `SERIOUS_PYTHON_BUNDLE_ID` pass now applies **only** to frameworks built here from the app's own wheels, which are generated in a separate directory, mutated there, and merged with the untouched provider artifacts afterwards. `reconcile_framework_install_names` (which rewrites Mach-O headers and re-signs ad-hoc) likewise never reaches a provider binary. +* **This retracts the reasoning in 4.4.2.** That entry claimed "a signature applied at the source can't matter", on the grounds that Xcode's `codesign -f` at embed and `exportArchive` overwrites it. The overwrite is real but the conclusion was wrong: Xcode *separately* records the state of each `.xcframework` as its publisher shipped it, and writes that into the IPA as `Signatures/.xcframework-ios.signature` with `signed` and `isSecureTimestamp` flags. `ITMS-91065: Missing signature` reports on **those receipts**, not on the embedded copy. Namespacing `CFBundleIdentifier` under the host app was therefore not a fix, and was actively counterproductive — the plist edit invalidated the very signature the receipt reports on. The 4.4.2 identifier rewrite is reverted for provider frameworks and kept for locally built ones, where it is harmless and matches CPython's convention. +* **New: provider integrity is checked at every staging step.** A SHA-256 manifest of the provider XCFrameworks is recorded when they are extracted, and re-checked after site-package synchronization and after SwiftPM staging. Any file that changed fails the build. `SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES` (`warn` default / `require` / `off`) additionally checks that each provider XCFramework carries a real, securely timestamped publisher signature; `SERIOUS_PYTHON_EXPECTED_TEAM_ID` pins the expected Apple Team ID. The default is `warn` so that pinning an older, pre-signing `python-build` or `dart-bridge` release still builds — use `require` for App Store submissions. +* **iOS: use the Swift Package Manager path for App Store submissions.** The CocoaPods path installs the stdlib and site-package extensions by copying the *inner* `.framework` bundles in a `Pods-Runner-frameworks.sh` script phase; Xcode writes SDK-origin receipts only for xcframeworks it consumes as declared binary dependencies, so that path cannot produce complete receipts no matter how the source is signed. It now emits an Xcode `warning:` saying so. The SwiftPM path declares them as `binaryTarget`s and is unaffected. +* Provider identifiers are now assigned upstream and are stable: `dev.flet.python.runtime` for the Python runtime, `dev.flet.python.` for each stdlib extension, `dev.flet.dartbridge` for the bridge. They no longer depend on the consuming app, which is what allows one publisher signature to cover every app. +* Re-pins the bundled python-build snapshot to **20260729** (`dart_bridge` **1.6.1 → 1.7.0**). No Python version moved — **3.12.13 / 3.13.14 / 3.14.6** and Pyodide are unchanged from 20260727. What did change is that every Apple XCFramework in that release, and `dart_bridge.xcframework` in 1.7.0, is now signed by the Flet publishing team's Apple Distribution identity with a secure timestamp: verified on the published artifacts as `Authority=Apple Distribution: Appveyor Systems Inc. (GXXRQJK434)`, a real `Timestamp=`, and an outer `_CodeSignature/CodeResources` — 56 XCFrameworks in `python-ios-dart-3.14.6`, plus the macOS and mobile-forge runtimes. See flet-dev/python-build#37 and flet-dev/dart-bridge#13. + +## 4.4.2 + +* **iOS: framework bundle identifiers are now namespaced under the host app.** Every Python C-extension ships as its own embedded framework, and each carried a fixed `org.python.` `CFBundleIdentifier` — `org.python.ssl`, `org.python.hashlib`, … — byte-identical in every app ever built with serious_python. Set `SERIOUS_PYTHON_BUNDLE_ID` to the app's bundle identifier and they become `com.example.myapp.-ssl` instead, matching what CPython's own iOS support does when it converts a `.so` into a framework. This matters because a framework's bundle identifier also becomes its **code signing identifier**, which is the one field that survives the `codesign -f` Xcode applies at embed and again at `exportArchive`. `Python.framework` is covered too — it was `org.python.python` in every app; nothing resolves it by identifier (the string appears in no shipped binary and there are no `CFBundleGetBundleWithIdentifier` lookups), so the rename is inert at runtime. `dart_bridge.framework` is deliberately left at `dev.flet.dartbridge`, which is already a vendor-owned identifier rather than a shared placeholder. Unset, or set to something that isn't a valid bundle identifier, leaves the old defaults in place and prints a warning. `flet build` supplies it from your app's configured bundle id (flet-dev/flet#6731). +* **iOS: generated frameworks now carry Xcode's build-provenance keys in their `Info.plist`.** Every framework built from a `lib-dynload` `.so` — `_ssl`, `_hashlib`, and the rest — previously shipped ten hand-written keys and not a single `DT*` key, so to Apple's static analysis the bundle didn't look like anything Xcode had produced. They now carry the same key set Xcode stamps into a real framework (`BuildMachineOSBuild`, `DTCompiler`, `DTPlatformBuild`, `DTPlatformName`, `DTPlatformVersion`, `DTSDKBuild`, `DTSDKName`, `DTXcode`, `DTXcodeBuild`, `UIDeviceFamily`, and `UIRequiredDeviceCapabilities` on the device slice), resolved from the SDK the release was built with. The simulator slice is also labelled correctly now — it previously claimed `CFBundleSupportedPlatforms=iPhoneOS`. See flet-dev/python-build#36. +* Both changes above are hypotheses under test for the `ITMS-91065: Missing signature` App Store rejection in flet-dev/flet#6724, **neither is a confirmed fix**. The shared bundle identifier is the stronger of the two: it accounts for every outcome we could find (BeeWare namespaces under the app and has no reports; a third-party OpenSSL xcframework with its own vendor identifier is reported to pass; our `org.python.*` and one other project's independent identifier both draw the rejection). Two further candidates were ruled out experimentally: a signature applied at the source can't matter (re-signing the way `exportArchive` does destroys the certificate chain, team identifier and timestamp, leaving only the identifier string, which is derived from `CFBundleIdentifier` for an unsigned framework anyway), and the privacy manifest can't be the differentiator (a third-party OpenSSL xcframework the App Store accepts ships an empty stub, and our frameworks already carry a manifest — Apple emits no `ITMS-91061` alongside the rejection). The check only fires for new apps, or updates that newly add a listed SDK, so existing published apps are unaffected either way. +* Re-pins the bundled python-build snapshot to **20260727**. No versions moved: Python (**3.12.13 / 3.13.14 / 3.14.6**), Pyodide, and `dart_bridge` **1.6.1** are unchanged from 20260726, and the `Info.plist` change above is the only functional difference on any platform. + +## 4.4.1 + +* **iOS: `_ssl.framework` and `_hashlib.framework` now carry OpenSSL's official privacy manifest.** Both extensions statically link OpenSSL, so Apple's App Store scan identifies them as containing BoringSSL / openssl_grpc — a listed third-party SDK that must declare a privacy manifest. The bundled `PrivacyInfo.xcprivacy` was a stub: `NSPrivacyAccessedAPITypes` was an empty array, and it carried an `NSPrivacyUsesNonStandardAPIs` key that isn't part of Apple's schema. It is now byte-identical to the manifest OpenSSL publishes for its Apple builds, declaring the file-timestamp API access the library actually performs (`NSPrivacyAccessedAPICategoryFileTimestamp`, reason `C617.1`). See flet-dev/python-build#35. +* This does **not** resolve the `ITMS-91065: Missing signature` App Store rejection reported in flet-dev/flet#6724, and shouldn't be read as a fix for it. That rejection cites the same two frameworks but arrives with no accompanying `ITMS-91061`, so Apple does find the manifest. Inspecting a built `.ipa` confirms the frameworks reach Apple signed — valid Apple Distribution authority and team ID, with the manifest sealed into `_CodeSignature/CodeResources` — and that Xcode re-signs every embedded framework with the submitting team's identity at both the embed and `exportArchive` steps, overwriting whatever signature the vendor applied. Root cause is still open; this change removes the one verifiable defect so it isn't a confound. +* Re-pins the bundled python-build snapshot to **20260726**. No versions moved: Python (**3.12.13 / 3.13.14 / 3.14.6**), Pyodide, and `dart_bridge` **1.6.1** are all unchanged from 20260725, and the privacy manifest above is the only functional difference on any platform. + +## 4.4.0 + +* **iOS/macOS: `dart_bridge` now ships as a dynamic framework**, fixing built iOS apps crashing at startup with `Failed to lookup symbol 'serious_python_run': dlsym(RTLD_DEFAULT, serious_python_run): symbol not found`. `dart_bridge`'s FFI entry points (`serious_python_run`, `DartBridge_InitDartApiDL`, `DartBridge_EnqueueMessage`, `PyInit_dart_bridge`) are resolved at runtime via `dlsym` — from Dart through `DynamicLibrary.process()` and from Python through `import dart_bridge`. It previously shipped as a **static** archive linked into the host app *executable*, and an iOS executable exports nothing to the dynamic symbol table by default (the release build also strips local symbols), so those lookups failed. Only release/archive (device) builds were affected — debug/simulator builds don't dead-strip, so the failure did not reproduce there. Android was never affected: its `dart_bridge` is a dynamic `.so`, which exports its symbols. `dart_bridge` **1.6.1** now builds `dart_bridge.xcframework` as a dynamic framework (see flet-dev/dart-bridge#11 and #12), so it is embedded + signed into the app like `Python.xcframework` and its symbols stay exported. The SwiftPM `-all_load` / `-force_load` retention, the CocoaPods `-all_load`, and the plugin's dead-strip keep-alive references are all removed — a loaded image needs none of them. +* Re-pins the bundled python-build snapshot to **20260725** (`dart_bridge` **1.5.1 → 1.6.1**). Python versions (**3.12.13 / 3.13.14 / 3.14.6**) are unchanged and the iOS/macOS CPython runtimes are byte-identical to 20260720. + +## 4.3.6 + +* Re-pins the bundled python-build snapshot to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**). 1.5.1 is a Windows-only UTF-8 startup fix (see `serious_python_windows` 4.3.6); the iOS/macOS runtimes are functionally unchanged from 20260719. + +## 4.3.4 + +* Re-pins the bundled python-build snapshot to **20260719** (previously 20260714). The snapshot un-prunes `_pyrepl` on Windows/Linux desktop (fixing a 3.14 `pydoc`/`pdb` crash — see `serious_python_windows` / `serious_python_linux` 4.3.4); iOS/macOS already shipped `_pyrepl` (un-pruned in 4.3.2), so these runtimes are byte-identical to 20260714. No iOS/macOS-affecting changes. + +## 4.3.3 + +* Version bump aligning with the `serious_python_*` 4.3.3 release (a Windows build fix). No iOS/macOS-affecting changes. + +## 4.3.2 + +* **iOS: reconcile framework install-names for interdependent bundled dylibs** ([#223](https://github.com/flet-dev/serious-python/issues/223)). Site-package `.so`/`.dylib`s are wrapped into frameworks named by their dotted relative path (`opt/lib/libarrow.dylib` → `opt.lib.libarrow.framework/opt.lib.libarrow`), but the Mach-O install-id and every interdependent `@rpath` reference were left at their original bare name (`@rpath/libarrow.dylib`). Because each framework is a `Package.swift` binaryTarget linked at launch, dyld could not resolve `@rpath/libarrow.dylib` (it looks for `Frameworks/libarrow.dylib`, which does not exist) and the app crashed **before Python started** — hitting any package that bundles a chain of interdependent libs (pyarrow's `libarrow`/`libarrow_compute`/`libarrow_python`, llama-cpp-python's `libggml*`/`libllama`). A new reconcile pass, run after `sync_site_packages` frameworks the libs, sets each framework's own install-id to `@rpath/.framework/` and rewrites every dependency pointing at a sibling's old id to that framework path, then re-signs. The Python/stdlib xcframeworks are left untouched. (This supersedes the 4.2.1 approach of preserving `.dylib` install-names, which only worked when every sibling happened to be loaded first.) +* The reconcile pass fails the build — rather than silently swallowing — on a genuine `install_name_tool`/`codesign` error, notably insufficient Mach-O header space to grow a load command (which would otherwise leave a bare `@rpath` ref and reproduce the launch crash), and records every framework slice's old install-id so a lib with divergent per-slice install names has all slices rewritten. +* Bump the bundled python-build snapshot to `20260714` (Python/`dart_bridge` versions are unchanged), with two iOS fixes: + * `_pyrepl` is no longer pruned from the bundled stdlib: Python **3.14**'s `pdb` imports `_pyrepl` at module load, so anything importing `pdb` (e.g. pytest's debugging plugin) died with `ModuleNotFoundError: No module named '_pyrepl'` on iOS (3.13's `pdb` doesn't import it). + * The `_posixshmem` extension is now built into the iOS runtimes: `multiprocessing.resource_tracker` — imported transitively by `import multiprocessing` (e.g. scikit-learn → joblib) — unconditionally imports it on posix, so with `_multiprocessing` enabled (since `20260701`) but `_posixshmem` missing, any `multiprocessing` user died with `ModuleNotFoundError: No module named '_posixshmem'`. Process *spawning* remains unsupported in the iOS sandbox — this only makes the shared-memory module importable. + +## 4.3.1 + +* Version bump aligning with the `serious_python_*` 4.3.1 release. + +## 4.3.0 + +* Bump `dart_bridge` to **1.5.0** (python-build snapshot `20260708`): multiprocessing child-interception exports (`serious_python_is_mp_invocation` / `serious_python_main`), kept alive against the host link's `-dead_strip` both by `__attribute__((used))` in the archive and by keep-alive references in `SeriousPythonPlugin.swift`. See the `serious_python` 4.3.0 notes. +* `prepare_macos.sh` / `prepare_ios.sh`: the extracted `dart_bridge.xcframework` in `dist_*` is now keyed to the dart_bridge version (`.dart_bridge_version` marker) — previously a version bump kept staging the stale extraction from the earlier version. +* `PYTHONINSPECT=1` is no longer set by any platform implementation. It had no effect on the embedded interpreter, but it leaked into the process environment where any *real* interpreter child (e.g. a serviced multiprocessing worker) would inherit it and hang in interactive mode after its command completed. + ## 4.2.1 * Framework-ize ctypes `.dylib` shared libs (not just `.so` C-extensions) when syncing iOS site-packages, so `.dylib`-shipping packages (e.g. `llama-cpp-python`) load on the **iOS simulator** instead of failing `dlopen` with `incompatible platform (have 'iOS', need 'iOS-simulator')`. Each `.dylib` becomes a device+simulator xcframework + `.fwork` pointer, exactly like `.so`; unlike `.so` (whose id is rewritten to the framework path), the `.dylib` install-name is preserved so multi-lib packages resolve their sibling libs. The `.so` path is unchanged. diff --git a/src/serious_python_darwin/darwin/bundle-python-frameworks-ios.sh b/src/serious_python_darwin/darwin/bundle-python-frameworks-ios.sh index 3c9766f0..04d0dd61 100755 --- a/src/serious_python_darwin/darwin/bundle-python-frameworks-ios.sh +++ b/src/serious_python_darwin/darwin/bundle-python-frameworks-ios.sh @@ -1,5 +1,16 @@ echo "Bundle Python xcframeworks" +# This path installs the stdlib / site-package extensions by copying the INNER +# .framework bundles out of each .xcframework (see pods-runner-frameworks-addon.sh). +# Xcode only writes an SDK-origin receipt — Signatures/.xcframework-ios.signature +# in the IPA — for xcframeworks it consumes as a declared binary dependency, so +# frameworks installed this way arrive with no provenance record regardless of how +# well the source xcframework was signed. That is what Apple reports as +# `ITMS-91065: Missing signature`. +# +# `warning:` is the prefix Xcode surfaces in the build log and issue navigator. +echo "warning: serious_python: the CocoaPods iOS path copies inner .framework bundles and cannot produce SDK-origin XCFramework signature receipts. Use the Swift Package Manager path for App Store submissions." + # modify Flutter packaging script to include user frameworks if [ -n "$PODS_TARGET_SRCROOT" ]; then echo "modify Flutter packaging script to include user frameworks: $PODS_TARGET_SRCROOT" diff --git a/src/serious_python_darwin/darwin/prepare_ios.sh b/src/serious_python_darwin/darwin/prepare_ios.sh index 279c99b8..02f571ae 100755 --- a/src/serious_python_darwin/darwin/prepare_ios.sh +++ b/src/serious_python_darwin/darwin/prepare_ios.sh @@ -15,7 +15,8 @@ cache_root="${FLET_CACHE_DIR:-$HOME/.flet/cache}" # date — e.g. a rebuild that only re-signs binaries) downloads fresh instead of # being served stale from the previous release's cache. dart-bridge stays # version-keyed (its re-releases bump the version). -pb_cache="$cache_root/python-build/v$python_full_version-$python_build_date" +pb_id="$python_full_version-$python_build_date" +pb_cache="$cache_root/python-build/v$pb_id" db_cache="$cache_root/dart-bridge/v$dart_bridge_version" mkdir -p "$pb_cache" "$db_cache" @@ -34,14 +35,13 @@ fi # 3.14 site-packages, which trips C-extension ABI errors ("unknown slot ID") at # import. The marker keys the extracted dist to the version + release date, so a # same-version re-release (new build date) also re-extracts. -build_id="$python_full_version-$python_build_date" marker="$dist/.python_build_id" -if [ ! -d "$dist" ] || [ "$(cat "$marker" 2>/dev/null)" != "$build_id" ]; then +if [ ! -d "$dist" ] || [ "$(cat "$marker" 2>/dev/null)" != "$pb_id" ]; then rm -rf "$dist" mkdir -p "$dist" tar -xzf "$python_ios_dist_path" -C "$dist" mv "$dist/python-stdlib" "$dist/stdlib" - echo "$build_id" > "$marker" + echo "$pb_id" > "$marker" fi # ---- flet-dev/dart-bridge (xcframework) ----------------------------------- @@ -55,7 +55,23 @@ if [ ! -f "$dart_bridge_path" ]; then mv "$dart_bridge_path.tmp" "$dart_bridge_path" fi -if [ ! -d "$dist/xcframeworks/dart_bridge.xcframework" ]; then +# Cache dart_bridge.xcframework by version; extract it if missing or if the version marker differs. +db_marker="$dist/xcframeworks/.dart_bridge_version" +if [ ! -d "$dist/xcframeworks/dart_bridge.xcframework" ] || [ "$(cat "$db_marker" 2>/dev/null)" != "$dart_bridge_version" ]; then + rm -rf "$dist/xcframeworks/dart_bridge.xcframework" mkdir -p "$dist/xcframeworks" unzip -q "$dart_bridge_path" -d "$dist/xcframeworks/" + echo "$dart_bridge_version" > "$db_marker" fi + +# ---- provider provenance --------------------------------------------------- +# Both provider archives are now extracted. Verify their signatures at the point +# of arrival (so a bad download is caught here, not in an App Store rejection), +# and record a digest manifest of everything they contain. Every later staging +# step re-checks that manifest: the provider bundles must reach the app exactly +# as their publisher signed them, because editing any file inside one destroys +# the SDK-origin signature that the IPA's Signatures/ receipts report on. +. "$script_dir/xcframework_verify.sh" +spv_verify_provider "$dist/xcframeworks" "$dist/python-xcframeworks" || exit 1 +spv_manifest_record "$dist/.provider-manifests/xcframeworks.sha256" "$dist/xcframeworks" +spv_manifest_record "$dist/.provider-manifests/python-xcframeworks.sha256" "$dist/python-xcframeworks" diff --git a/src/serious_python_darwin/darwin/prepare_macos.sh b/src/serious_python_darwin/darwin/prepare_macos.sh index 251c0a39..ea4ebf50 100755 --- a/src/serious_python_darwin/darwin/prepare_macos.sh +++ b/src/serious_python_darwin/darwin/prepare_macos.sh @@ -16,7 +16,8 @@ cache_root="${FLET_CACHE_DIR:-$HOME/.flet/cache}" # date — e.g. a rebuild that only re-signs binaries) downloads fresh instead of # being served stale from the previous release's cache. dart-bridge stays # version-keyed (its re-releases bump the version). -pb_cache="$cache_root/python-build/v$python_full_version-$python_build_date" +pb_id="$python_full_version-$python_build_date" +pb_cache="$cache_root/python-build/v$pb_id" db_cache="$cache_root/dart-bridge/v$dart_bridge_version" mkdir -p "$pb_cache" "$db_cache" @@ -36,9 +37,8 @@ fi # 3.14 site-packages, which trips C-extension ABI errors ("unknown slot ID") at # import. The marker keys the extracted dist to the version + release date, so a # same-version re-release (new build date) also re-extracts. -build_id="$python_full_version-$python_build_date" marker="$dist/.python_build_id" -if [ ! -d "$dist" ] || [ "$(cat "$marker" 2>/dev/null)" != "$build_id" ]; then +if [ ! -d "$dist" ] || [ "$(cat "$marker" 2>/dev/null)" != "$pb_id" ]; then rm -rf "$dist" mkdir -p "$dist" tar -xzf "$python_macos_dist_path" -C "$dist" @@ -54,7 +54,7 @@ if [ ! -d "$dist" ] || [ "$(cat "$marker" 2>/dev/null)" != "$build_id" ]; then # unexpectedly" crash dialog. We don't need this launcher for embedded # use; libdart_bridge dlopens Python.framework's main binary directly. find "$dist/xcframeworks" -type d -name 'Python.app' -prune -exec rm -rf {} + - echo "$build_id" > "$marker" + echo "$pb_id" > "$marker" fi # ---- flet-dev/dart-bridge (xcframework, same archive for macOS + iOS) ----- @@ -68,7 +68,24 @@ if [ ! -f "$dart_bridge_path" ]; then mv "$dart_bridge_path.tmp" "$dart_bridge_path" fi -if [ ! -d "$dist/xcframeworks/dart_bridge.xcframework" ]; then +# Cache dart_bridge.xcframework by version; extract it if missing or if the version marker differs. +db_marker="$dist/xcframeworks/.dart_bridge_version" +if [ ! -d "$dist/xcframeworks/dart_bridge.xcframework" ] || [ "$(cat "$db_marker" 2>/dev/null)" != "$dart_bridge_version" ]; then + rm -rf "$dist/xcframeworks/dart_bridge.xcframework" mkdir -p "$dist/xcframeworks" unzip -q "$dart_bridge_path" -d "$dist/xcframeworks/" + echo "$dart_bridge_version" > "$db_marker" fi + +# ---- provider provenance --------------------------------------------------- +# Runs last, and in particular AFTER the Python.app removal above. That removal +# is a legacy safety net -- python-build has excluded Python.app from the shipped +# framework since the exclude list gained it, so on any current artifact it +# matches nothing -- but it does delete a file from inside a provider bundle, so +# it has to happen before the signature check and the digest snapshot rather +# than after. Nothing downstream of this point may modify these bundles: editing +# any file inside a signed XCFramework destroys the SDK-origin signature that the +# IPA's Signatures/ receipts report on. +. "$script_dir/xcframework_verify.sh" +spv_verify_provider "$dist/xcframeworks" || exit 1 +spv_manifest_record "$dist/.provider-manifests/xcframeworks.sha256" "$dist/xcframeworks" diff --git a/src/serious_python_darwin/darwin/python_versions.properties b/src/serious_python_darwin/darwin/python_versions.properties index faf98a2e..0bdc3bba 100644 --- a/src/serious_python_darwin/darwin/python_versions.properties +++ b/src/serious_python_darwin/darwin/python_versions.properties @@ -1,8 +1,8 @@ # GENERATED by `dart run serious_python:gen_version_tables` from -# python-build manifest.json (release 20260701). Do not edit by hand. +# python-build manifest.json (release 20260730). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.4.1 -python_build_release_date=20260701 +dart_bridge_version=1.7.1 +python_build_release_date=20260730 3.12.full_version=3.12.13 3.12.android_abis=arm64-v8a,x86_64,armeabi-v7a 3.13.full_version=3.13.14 diff --git a/src/serious_python_darwin/darwin/serious_python_darwin.podspec b/src/serious_python_darwin/darwin/serious_python_darwin.podspec index 83e93967..6e4ef60e 100644 --- a/src/serious_python_darwin/darwin/serious_python_darwin.podspec +++ b/src/serious_python_darwin/darwin/serious_python_darwin.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'serious_python_darwin' - s.version = '4.2.1' + s.version = '4.5.1' s.summary = 'A cross-platform plugin for adding embedded Python runtime to your Flutter apps.' s.description = <<-DESC A cross-platform plugin for adding embedded Python runtime to your Flutter apps. @@ -13,11 +13,10 @@ Pod::Spec.new do |s| s.license = { :file => '../LICENSE' } s.author = { 'Appveyor Systems Inc.' => 'hello@flet.dev' } s.source = { :path => '.' } - # dart_bridge.xcframework (vendored below) contains static .a archives. - # CocoaPods 1.16+ refuses to install vendored xcframeworks with static - # libraries unless the consuming pod is itself declared as a static - # framework. Python.xcframework is also static, so this was always - # implicitly the case — just being explicit now. + # This pod's own Swift code links as a static framework. The vendored + # xcframeworks below (Python, dart_bridge) are dynamic frameworks — they are + # embedded and signed into the host app, and their symbols stay exported so + # the dlsym lookups Dart and Python perform at runtime resolve. s.static_framework = true s.source_files = ['serious_python_darwin/Sources/serious_python_darwin/**/*.swift'] s.ios.dependency 'Flutter' @@ -26,9 +25,12 @@ Pod::Spec.new do |s| s.osx.deployment_target = '11.0' # Flutter.framework does not contain a i386 slice. + # `-all_load` is no longer needed: it existed to retain the static + # libdart_bridge.a against -dead_strip, and dart_bridge now ships as a dynamic + # framework whose symbols are exported from its own loaded image. s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', - 'OTHER_LDFLAGS' => '-ObjC -all_load -lc++' + 'OTHER_LDFLAGS' => '-ObjC -lc++' } s.swift_version = '5.0' diff --git a/src/serious_python_darwin/darwin/serious_python_darwin/Package.swift b/src/serious_python_darwin/darwin/serious_python_darwin/Package.swift index f6988e9a..1c0bfcf5 100644 --- a/src/serious_python_darwin/darwin/serious_python_darwin/Package.swift +++ b/src/serious_python_darwin/darwin/serious_python_darwin/Package.swift @@ -44,36 +44,20 @@ func staged(_ rel: String) -> Bool { var binaryTargets: [Target] = [] var deps: [Target.Dependency] = [.product(name: "FlutterFramework", package: "FlutterFramework")] -// dart_bridge: static archive, link-only. Its FFI exports are resolved at runtime -// via dlsym (Dart's `DynamicLibrary.process()` and Python's `import dart_bridge`), -// so the whole archive must be retained against the linker's -dead_strip. We -// force-load JUST this archive rather than using a global -all_load: SwiftPM -// applies a target's linkerSettings to the FINAL app executable link, so a global -// -all_load there would force-load EVERY static archive on the link line — -// including plugins that statically bundle the same third-party code (e.g. -// flet-video/media_kit and flet-rive both embed miniaudio), which then collide -// with thousands of duplicate symbols. -force_load on dart_bridge alone mirrors -// what the CocoaPods podspec did via `pod_target_xcconfig` (scoped to this pod's -// own inputs — only dart_bridge among them is a static archive). On macOS the -// xcframework has a single universal slice with a stable path, so we resolve it -// here. On iOS, plugin pods link as dynamic frameworks (no static-archive -// collision) and device/simulator use different slice paths, so the -// slice-agnostic -all_load is kept there (see linkerSettings below). -var macosDartBridgeForceLoad: [String] = [] +// dart_bridge: DYNAMIC framework -> embedded + auto-signed in the app. Its FFI +// entry points (serious_python_run, DartBridge_*, PyInit_dart_bridge) are +// EXPORTED from the loaded framework, so the dlsym lookups that Dart +// (`DynamicLibrary.process()`) and Python (`import dart_bridge`) perform resolve +// at runtime. A static library would instead be linked into the app executable, +// which does not export its symbols (and the release build strips them), so +// dlsym would fail with "Failed to lookup symbol 'serious_python_run'". Shipping +// it dynamic mirrors the Android `.so` (CMake SHARED) that already works — see +// flet-dev/dart-bridge apple/build_xcframework.sh. No -all_load/-force_load or +// keep-alive is needed anymore: the framework is a loaded image, not archive +// members that must be pulled in and kept against -dead_strip. if staged("dart_bridge.xcframework") { binaryTargets.append(.binaryTarget(name: "dart_bridge", path: "dart_bridge.xcframework")) deps.append("dart_bridge") - let xc = pkgDir.appendingPathComponent("dart_bridge.xcframework") - if let slices = try? FileManager.default.contentsOfDirectory( - at: xc, includingPropertiesForKeys: nil) { - for slice in slices.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) - where slice.lastPathComponent.hasPrefix("macos") { - let lib = slice.appendingPathComponent("libdart_bridge.a") - if FileManager.default.fileExists(atPath: lib.path) { - macosDartBridgeForceLoad = ["-force_load", lib.path] - } - } - } } // Python.framework: dynamic -> embedded + auto-signed. iOS and macOS ship separate // xcframeworks, so each is platform-conditional. @@ -119,15 +103,9 @@ let package = Package( .copy("Resources/app"), ], linkerSettings: [ + // dart_bridge is now a dynamic framework (embedded, symbols + // exported), so no -all_load / -force_load is needed to retain it. .unsafeFlags(["-ObjC"]), - // iOS: slice-agnostic -all_load. Plugin pods link as dynamic - // frameworks, so dart_bridge is the only static archive in scope — - // nothing else to double-load. (device/simulator slice paths differ, - // so a single -force_load path isn't possible here.) - .unsafeFlags(["-all_load"], .when(platforms: [.iOS])), - // macOS: force-load ONLY dart_bridge (resolved above), never a global - // -all_load — see the note where macosDartBridgeForceLoad is built. - .unsafeFlags(macosDartBridgeForceLoad, .when(platforms: [.macOS])), .linkedLibrary("c++"), ] ), diff --git a/src/serious_python_darwin/darwin/serious_python_darwin/Sources/serious_python_darwin/SeriousPythonPlugin.swift b/src/serious_python_darwin/darwin/serious_python_darwin/Sources/serious_python_darwin/SeriousPythonPlugin.swift index d676aa0b..b24022ed 100644 --- a/src/serious_python_darwin/darwin/serious_python_darwin/Sources/serious_python_darwin/SeriousPythonPlugin.swift +++ b/src/serious_python_darwin/darwin/serious_python_darwin/Sources/serious_python_darwin/SeriousPythonPlugin.swift @@ -5,16 +5,13 @@ import UIKit import FlutterMacOS #endif -// Keep-alive references to dart_bridge.xcframework's C entry points. Dart -// resolves them via `DynamicLibrary.process()` at runtime; without these -// static references the host app's linker `-dead_strip` pass could drop -// them even though `-all_load` pulled the archive members in. -@_silgen_name("serious_python_run") -private func _sp_run_keepalive(_ cfg: OpaquePointer?) -> Int32 -@_silgen_name("DartBridge_InitDartApiDL") -private func _sp_init_keepalive(_ data: UnsafeMutableRawPointer?) -> Int -@_silgen_name("DartBridge_EnqueueMessage") -private func _sp_enqueue_keepalive(_ data: UnsafePointer?, _ len: Int) +// dart_bridge's C entry points (serious_python_run, DartBridge_*, +// serious_python_{is_mp_invocation,main}) used to need keep-alive references +// here: it shipped as a static archive linked into the host app executable, +// which exports nothing, so the dlsym lookups Dart (`DynamicLibrary.process()`) +// and the macOS host's main.swift perform could fail. dart_bridge now ships as a +// dynamic framework (embedded + signed) whose symbols are exported from its own +// loaded image, so no keep-alive is required — see dart-bridge 1.6.0. /// Thin Flutter plugin: surfaces the python.bundle resource path to Dart. /// All Python lifecycle now lives in `serious_python_run` @@ -31,15 +28,6 @@ public class SeriousPythonPlugin: NSObject, FlutterPlugin { let channel = FlutterMethodChannel(name: "serious_python", binaryMessenger: messenger) let instance = SeriousPythonPlugin() registrar.addMethodCallDelegate(instance, channel: channel) - - // Static reference to dart_bridge symbols — see top-of-file note. - // The branch is unreachable at runtime (the registrar argument is - // always non-nil), but the linker only sees a live call site. - if unsafeBitCast(registrar, to: Int.self) == 0 { - _ = _sp_run_keepalive(nil) - _ = _sp_init_keepalive(nil) - _sp_enqueue_keepalive(nil, 0) - } } public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { diff --git a/src/serious_python_darwin/darwin/stage_spm.sh b/src/serious_python_darwin/darwin/stage_spm.sh index 0ebba398..06e9296f 100755 --- a/src/serious_python_darwin/darwin/stage_spm.sh +++ b/src/serious_python_darwin/darwin/stage_spm.sh @@ -18,6 +18,9 @@ res="$pkg/Sources/serious_python_darwin/Resources" [ -d "$dist" ] || { echo "stage_spm: $dist not found" >&2; exit 1; } +# shellcheck source=src/serious_python_darwin/darwin/xcframework_verify.sh +. "$script_dir/xcframework_verify.sh" + # 1. Python runtime (dynamic framework -> embedded). Platform-specific path so a # single shared manifest can carry both via platform-conditional binaryTargets. rm -rf "$pkg/Python-$platform.xcframework" @@ -44,6 +47,41 @@ for tree in stdlib site-packages app; do [ -d "$dist/$tree" ] && rsync -a --exclude '.pod' "$dist/$tree/" "$dest/" done +# 4b. Provenance gate. This is the last place serious_python touches the provider +# XCFrameworks before Xcode consumes them as binaryTargets and writes the +# IPA's Signatures/ receipts, so it is the last chance to catch a mutation. +# The staged copies are compared against the digest manifest recorded at +# extraction time; `cp -R` preserves content, symlinks, and _CodeSignature, +# so an identical manifest here means the SDK-origin signature survived. +# +# Python-.xcframework is a renamed copy, so it is checked against +# the manifest entries for its source name rather than by path. +# +# Everything in this block is redirected to stderr: this script's stdout is +# the SP_NATIVE_SET key, which prepare_spm.sh captures verbatim. +{ + python_manifest="$dist/.provider-manifests/Python.xcframework.sha256" + spv_manifest_record "$python_manifest" "$dist/xcframeworks/Python.xcframework" + spv_manifest_check "$python_manifest" "$pkg/Python-$platform.xcframework" \ + "staged Python-$platform.xcframework" + + dart_bridge_manifest="$dist/.provider-manifests/dart_bridge.xcframework.sha256" + spv_manifest_record "$dart_bridge_manifest" "$dist/xcframeworks/dart_bridge.xcframework" + spv_manifest_check "$dart_bridge_manifest" "$pkg/dart_bridge.xcframework" \ + "staged dart_bridge.xcframework" + + if [ "$platform" = "ios" ] && [ -d "$pkg/extra-xcframeworks" ]; then + # extra-xcframeworks mixes provider stdlib frameworks with frameworks + # built locally from the app's own wheels. spv_manifest_check ignores + # files the manifest doesn't name, so this validates exactly the + # provider subset. + spv_manifest_check "$dist/.provider-manifests/python-xcframeworks.sha256" \ + "$pkg/extra-xcframeworks" "staged extra-xcframeworks (provider subset)" + fi + + spv_verify_provider "$pkg/Python-$platform.xcframework" "$pkg/dart_bridge.xcframework" +} >&2 + # 5. Cache-bust key: platform + Python version + the staged native/resource set # (path+size). Changes whenever requirements, app, or Python version change. key_paths=("Python-$platform.xcframework" "Sources/serious_python_darwin/Resources") diff --git a/src/serious_python_darwin/darwin/sync_site_packages.sh b/src/serious_python_darwin/darwin/sync_site_packages.sh index b02652ad..ed065943 100755 --- a/src/serious_python_darwin/darwin/sync_site_packages.sh +++ b/src/serious_python_darwin/darwin/sync_site_packages.sh @@ -1,5 +1,9 @@ script_dir=$(cd "$(dirname "$0")" && pwd -P) +# Provider-signature + provider-integrity checks (see xcframework_verify.sh). +# Sourced unconditionally so both the iOS and macOS branches can use it. +source $script_dir/xcframework_verify.sh + # App sources are arch- and platform-independent; stage them as a bare `app/` # resource bundle into BOTH dist trees, regardless of whether site-packages # exist (an app may have no pip dependencies). The per-target resource bundle @@ -21,14 +25,15 @@ if [[ -n "$SERIOUS_PYTHON_SITE_PACKAGES" && -d "$SERIOUS_PYTHON_SITE_PACKAGES" ] echo "Sync iOS xcframeworks and site-packages" dist=$script_dir/dist_ios - # app xcframeworks - rm -rf $dist/site-xcframeworks - mkdir -p $dist/site-xcframeworks - cp -R $dist/python-xcframeworks/* $dist/site-xcframeworks - source $script_dir/xcframework_utils.sh tmp_dir=$(mktemp -d) + # Locally generated frameworks are built and mutated HERE, in their own + # directory, never alongside the provider-built ones. Every app-specific + # rewrite below (bundle ids, install names, ad-hoc re-sign) is confined + # to this tree; the provider artifacts are merged in afterwards, + # byte-for-byte, and are never opened for writing. + local_xcframeworks=$(mktemp -d) cp -R $SERIOUS_PYTHON_SITE_PACKAGES/* $tmp_dir @@ -45,8 +50,57 @@ if [[ -n "$SERIOUS_PYTHON_SITE_PACKAGES" && -d "$SERIOUS_PYTHON_SITE_PACKAGES" ] $dylib_relative_path \ "$_sp_ext" \ "Frameworks/serious_python_darwin.framework/python.bundle/site-packages" \ - $dist/site-xcframeworks + $local_xcframeworks + done done + + # Namespace the LOCALLY BUILT frameworks' CFBundleIdentifiers under the + # host app's bundle id (see xcframework_utils.sh). These are created + # moments ago from this app's own wheels; nobody has signed them, so + # editing their plists invalidates nothing. + # + # This deliberately no longer touches Python.xcframework or the stdlib + # extension frameworks. Those arrive already signed by their provider, + # and a plist rewrite destroys that signature -- which is what made the + # IPA's Signatures/ receipts report `signed = false` and produced + # ITMS-91065. They now carry stable `dev.flet.python.*` identifiers + # assigned upstream in python-build, so there is nothing left to fix + # here anyway. + if [[ -n "${SERIOUS_PYTHON_BUNDLE_ID:-}" ]]; then + echo "Namespacing framework bundle identifiers under $SERIOUS_PYTHON_BUNDLE_ID" + rewrite_framework_bundle_ids "$local_xcframeworks" "$SERIOUS_PYTHON_BUNDLE_ID" || exit 1 + fi + + # After every .so/.dylib is framework-ized, reconcile the Mach-O + # install names so the interdependent @rpath refs point at the new + # framework paths (serious-python #223). Without this, dyld cannot + # resolve e.g. @rpath/libarrow.dylib at launch and the app crashes + # before Python starts. + # A reconcile failure (e.g. a Mach-O with no header space to grow a load + # command, or a signing error) would otherwise ship an app that crashes + # at launch -- abort the build instead. sync_site_packages.sh has no + # set -e; prepare_spm.sh runs it under set -euo pipefail, so a non-zero + # exit here fails `flet build` loudly. + # + # The exclude argument still names the provider stdlib directory. It is + # now belt and braces -- $local_xcframeworks holds only locally built + # frameworks -- but it keeps the guarantee explicit: this pass rewrites + # Mach-O headers and re-signs ad-hoc, and must never reach a provider + # binary. + reconcile_framework_install_names "$local_xcframeworks" "$dist/python-xcframeworks" || exit 1 + + # Assemble the staging directory: provider frameworks first, copied + # verbatim, then the locally built ones merged over the top (a name + # collision resolves to the local build, as it always has). The rm -rf + # in the loop is what makes that an overwrite rather than a `cp -R` + # merge, which would leave the provider's files behind inside it. + rm -rf $dist/site-xcframeworks + mkdir -p $dist/site-xcframeworks + cp -R $dist/python-xcframeworks/* $dist/site-xcframeworks + for _local_xcf in "$local_xcframeworks"/*.xcframework; do + [ -d "$_local_xcf" ] || continue + rm -rf "$dist/site-xcframeworks/$(basename "$_local_xcf")" + cp -R "$_local_xcf" "$dist/site-xcframeworks/" done rm -rf $dist/site-packages @@ -54,7 +108,17 @@ if [[ -n "$SERIOUS_PYTHON_SITE_PACKAGES" && -d "$SERIOUS_PYTHON_SITE_PACKAGES" ] cp -R $tmp_dir/${archs[0]}/* $dist/site-packages # cleanup - rm -rf "${tmp_dir}" >/dev/null + rm -rf "${tmp_dir}" "${local_xcframeworks}" >/dev/null + + # The provider artifacts must be bit-identical to what was extracted in + # prepare_ios.sh, both where they live and in the copies just staged. + spv_manifest_check "$dist/.provider-manifests/xcframeworks.sha256" \ + "$dist/xcframeworks" "dist_ios/xcframeworks" || exit 1 + spv_manifest_check "$dist/.provider-manifests/python-xcframeworks.sha256" \ + "$dist/python-xcframeworks" "dist_ios/python-xcframeworks" || exit 1 + spv_manifest_check "$dist/.provider-manifests/python-xcframeworks.sha256" \ + "$dist/site-xcframeworks" "dist_ios/site-xcframeworks (provider subset)" || exit 1 + spv_verify_provider "$dist/xcframeworks" "$dist/python-xcframeworks" || exit 1 else @@ -72,5 +136,12 @@ if [[ -n "$SERIOUS_PYTHON_SITE_PACKAGES" && -d "$SERIOUS_PYTHON_SITE_PACKAGES" ] # file. .pod is only needed by package_command.dart at packaging # time to invoke this sync script; it does not belong in the bundle. rsync -av --delete --exclude '.pod' "$SERIOUS_PYTHON_SITE_PACKAGES/" "$dist/site-packages/" + + # macOS has no framework-ization step -- its .so's load flat from the + # resource tree -- so nothing here should ever touch the provider + # xcframeworks. Assert it rather than assume it. + spv_manifest_check "$dist/.provider-manifests/xcframeworks.sha256" \ + "$dist/xcframeworks" "dist_macos/xcframeworks" || exit 1 + spv_verify_provider "$dist/xcframeworks" || exit 1 fi fi diff --git a/src/serious_python_darwin/darwin/xcframework_utils.sh b/src/serious_python_darwin/darwin/xcframework_utils.sh index 49a7b621..95c4bbde 100644 --- a/src/serious_python_darwin/darwin/xcframework_utils.sh +++ b/src/serious_python_darwin/darwin/xcframework_utils.sh @@ -104,4 +104,167 @@ create_xcframework_from_dylibs() { # cleanup popd >/dev/null rm -rf "${dylib_tmp_dir}" >/dev/null +} + +# Namespace LOCALLY BUILT frameworks' CFBundleIdentifiers under the host app's +# bundle id, mirroring what CPython's own iOS support does when it converts a +# .so into a framework (Platforms/Apple/testbed/Python.xcframework/build/utils.sh): +# +# FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") +# +# SCOPE -- read this before adding a call site. +# This may only be pointed at frameworks create_xcframework_from_dylibs built +# moments earlier from the app's own site-packages. It must NEVER be pointed at a +# provider-built artifact (Python.xcframework, dart_bridge.xcframework, or the +# stdlib extension frameworks from python-build). +# +# It used to be. The reasoning was that a globally-shared `org.python.` +# identifier on a framework Apple fingerprints as a third-party SDK explained the +# ITMS-91065 rejection in flet-dev/flet#6724. That hypothesis was wrong, and the +# fix was actively harmful: rewriting an Info.plist inside a provider XCFramework +# invalidates the provider's signature, and it is precisely that SDK-origin +# signature which ITMS-91065 is reporting missing. Xcode records it in the IPA's +# top-level Signatures/ receipts, separately from -- and unaffected by -- the app's +# own signature applied at embed and exportArchive. +# +# Provider frameworks now arrive with stable, provider-owned `dev.flet.python.*` +# identifiers assigned upstream in python-build (dart_bridge has always used +# `dev.flet.dartbridge`), so there is nothing here left to fix. +# +# The leading hyphen this produces for underscore-prefixed modules (`_ssl` -> +# `.-ssl`) is deliberate: it is what keeps `_ssl` distinct from `ssl`, and it +# is the exact form CPython and BeeWare ship. +# +# Must run BEFORE reconcile_framework_install_names, whose ad-hoc re-sign reseals +# the modified Info.plists. +# +# $3 is an optional space-separated list of framework names to leave alone. No +# current caller passes it; it is kept because the guarantee it encodes -- some +# names are off limits -- is cheaper to keep than to re-derive. +rewrite_framework_bundle_ids() { + local xcframeworks_dir=$1 + local bundle_id=${2%.} + local skip=" ${3:-} " + + # CFBundleIdentifier allows only [A-Za-z0-9.-], and every dot-separated + # component must be non-empty. The bundle id comes from the app's pyproject / + # CLI, so validate rather than emit a plist that fails much later at export + # with an error naming a framework the developer has never heard of. + if ! printf '%s' "$bundle_id" | grep -Eq '^[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*$'; then + echo "rewrite_framework_bundle_ids: '$bundle_id' is not a valid bundle identifier;" \ + "leaving framework identifiers at their org.python.* defaults" >&2 + return 0 + fi + + local xcf fw component identifier plist err + for xcf in "$xcframeworks_dir"/*.xcframework; do + [ -d "$xcf" ] || continue + fw=$(basename "$xcf" .xcframework) + case "$skip" in *" $fw "*) continue ;; esac + # Module names come from whatever wheels the app depends on, so anything + # outside the allowed set becomes a hyphen. + component=$(printf '%s' "$fw" | tr '_' '-' | sed 's/[^A-Za-z0-9.-]/-/g') + identifier="$bundle_id.$component" + for plist in "$xcf"/*/"$fw.framework/Info.plist"; do + [ -f "$plist" ] || continue + if ! err=$(plutil -replace CFBundleIdentifier -string "$identifier" "$plist" 2>&1); then + echo "rewrite_framework_bundle_ids: plutil failed for $plist: $err" >&2 + return 1 + fi + done + done +} + +# Reconcile install names across the newly-created site-package frameworks. +# +# create_xcframework_from_dylibs renames each lib to a framework named by its +# dotted relative path (opt/lib/libarrow.dylib -> opt.lib.libarrow.framework/ +# opt.lib.libarrow), but leaves the Mach-O install-id and every interdependent +# @rpath reference at their ORIGINAL bare names (e.g. @rpath/libarrow.dylib): +# it only rewrites the install-id, and only for ext=so. dyld links every one of +# these frameworks at app launch (each is a Package.swift binaryTarget the +# plugin depends on), so a bare @rpath/libarrow.dylib resolves to +# Frameworks/libarrow.dylib -- which does not exist -- and the app crashes +# BEFORE Python starts. See serious-python #223. +# +# This pass makes the install names match the framework layout: +# 1. set every framework binary's own install-id to @rpath/.framework/ +# 2. rewrite every dep that pointed at a sibling's OLD id to that sibling's +# framework path, so interdependent libs (libarrow_python -> libarrow, or a +# C-extension -> its bundled .dylib) resolve at launch. +# Only the frameworks created from site-packages are touched; the Python / +# stdlib xcframeworks (passed as $2) are already correct and left untouched. +reconcile_framework_install_names() { + local xcframeworks_dir=$1 + local exclude_dir=$2 + + local -a map_old=() + local -a map_new=() + + # Pass 1: set each framework's own install-id to @rpath/.framework/, + # and record old-id -> new-id for the dep rewrite below. Read the old id from + # EVERY slice, not just the first: a lib whose slices carry divergent install + # names (e.g. an upstream build that baked an arch-specific absolute path + # instead of an @rpath id) would otherwise leave the other slices' deps + # unrewritten. Distinct old ids all map to the same new id. + local xcf fw newid bin oldid raw err j found + for xcf in "$xcframeworks_dir"/*.xcframework; do + [ -d "$xcf" ] || continue + fw=$(basename "$xcf" .xcframework) + [ -n "$exclude_dir" ] && [ -e "$exclude_dir/$fw.xcframework" ] && continue + newid="@rpath/$fw.framework/$fw" + for bin in "$xcf"/*/"$fw.framework/$fw"; do + [ -f "$bin" ] || continue + # Buffer otool output before filtering: piping otool straight into + # `head -1` lets head close the pipe early, and the SIGPIPE race + # intermittently drops the first read (empty oldid). + raw=$(otool -D "$bin" 2>/dev/null) + oldid=$(printf '%s\n' "$raw" | grep -v ':$' | grep -vi 'Architectures in' | head -1 | sed 's/^[[:space:]]*//') + # -id must succeed; a failure means the binary is unwritable/corrupt + # and the app would crash at launch, so surface it instead of hiding + # it behind `|| true` (stderr is captured and only printed on error, + # to keep the expected "will invalidate the code signature" warning + # off the build log). + if ! err=$(install_name_tool -id "$newid" "$bin" 2>&1); then + echo "reconcile_framework_install_names: install_name_tool -id failed for $bin: $err" >&2 + return 1 + fi + if [ -n "$oldid" ] && [ "$oldid" != "$newid" ]; then + found=0 + for j in ${map_old[@]+"${map_old[@]}"}; do + [ "$j" = "$oldid" ] && { found=1; break; } + done + [ "$found" -eq 0 ] && { map_old+=("$oldid"); map_new+=("$newid"); } + fi + done + done + + # Pass 2: rewrite each binary's deps that point at a sibling's old id to that + # sibling's framework path, then re-sign (install_name_tool invalidates the + # ad-hoc signature). `install_name_tool -change` is a no-op returning 0 when + # the dep is absent, so a NON-zero rc means a dep that IS present could not be + # rewritten -- usually no Mach-O header space to grow the load command. That + # is fatal: it would leave a bare @rpath ref and reproduce the exact launch + # crash this pass exists to prevent, so fail the build rather than ship it. + local i n=${#map_old[@]} + for xcf in "$xcframeworks_dir"/*.xcframework; do + [ -d "$xcf" ] || continue + fw=$(basename "$xcf" .xcframework) + [ -n "$exclude_dir" ] && [ -e "$exclude_dir/$fw.xcframework" ] && continue + for bin in "$xcf"/*/"$fw.framework/$fw"; do + [ -f "$bin" ] || continue + i=0 + while [ $i -lt $n ]; do + if ! err=$(install_name_tool -change "${map_old[$i]}" "${map_new[$i]}" "$bin" 2>&1); then + echo "reconcile_framework_install_names: install_name_tool -change '${map_old[$i]}' -> '${map_new[$i]}' failed for $bin (no Mach-O header space?): $err" >&2 + return 1 + fi + i=$((i+1)) + done + if ! err=$(codesign --force --sign - "$bin" 2>&1); then + echo "reconcile_framework_install_names: codesign failed for $bin: $err" >&2 + return 1 + fi + done + done } \ No newline at end of file diff --git a/src/serious_python_darwin/darwin/xcframework_verify.sh b/src/serious_python_darwin/darwin/xcframework_verify.sh new file mode 100644 index 00000000..abafce85 --- /dev/null +++ b/src/serious_python_darwin/darwin/xcframework_verify.sh @@ -0,0 +1,258 @@ +#!/usr/bin/env bash +# +# Consumer-side integrity checks for the XCFrameworks serious_python does NOT +# build: Python.xcframework and the stdlib extension frameworks from +# flet-dev/python-build, and dart_bridge.xcframework from flet-dev/dart-bridge. +# +# WHY +# Xcode writes an SDK-origin receipt for every XCFramework an app links +# against into the IPA's `Signatures/` directory, reporting whether the +# XCFramework AS SHIPPED BY ITS PROVIDER was signed and carried a secure +# timestamp. Signing the app does not fill that in, and any edit to a file +# inside a signed XCFramework destroys it — which is exactly what this plugin +# used to do when it rewrote CFBundleIdentifier in the provider bundles. +# +# So this file exists to answer two questions at each staging step: +# 1. does the provider signature still verify? (spv_verify_provider) +# 2. did anything inside the provider bundles change? (spv_manifest_*) +# +# Question 2 matters even while the upstream artifacts are still unsigned: the +# digest manifest catches a reintroduced mutation immediately, rather than in +# an App Store rejection three releases later. +# +# MODE — SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES +# warn (default) report problems, do not fail the build +# require a missing or invalid provider signature fails the build +# off skip signature checks entirely +# +# The default is `warn` because a consumer may pin an older, pre-signing +# python-build / dart-bridge release through SERIOUS_PYTHON_BUILD_DATE or +# DART_BRIDGE_VERSION, and that must keep building. serious_python's own CI +# sets `require`. +# +# Digest-manifest checks are NOT gated by this: they cover code in this repo, +# which is always expected to leave provider bundles untouched. +# +# SERIOUS_PYTHON_EXPECTED_TEAM_ID +# When set, the provider signature's TeamIdentifier must match it. +# +# Written to POSIX sh so the podspec's prepare_command (run through /bin/sh) can +# source it as-is. + +spv_mode() { echo "${SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES:-warn}"; } + +spv_note() { echo "provider-verify: $*"; } + +# Report one problem. +# +# In `require` mode every problem is printed and the caller fails. In `warn` mode +# only the first few are printed and the rest are counted: a build against a +# pre-signing python-build release has ~57 unsigned xcframeworks, and 57 +# identical lines in every `flet build` teaches people to ignore the message. +# The summary line at the end always reports the true total. +spv_bad() { + _spv_problems=$((${_spv_problems:-0} + 1)) + if [ "$(spv_mode)" = "require" ] || [ "$_spv_problems" -le 3 ]; then + echo "provider-verify: $*" >&2 + elif [ "$_spv_problems" -eq 4 ]; then + echo "provider-verify: ... further problems suppressed;" \ + "set SERIOUS_PYTHON_VERIFY_PROVIDER_SIGNATURES=require to see them all" >&2 + fi + # Written as an `if` rather than `[ ] && return 1` so a caller running under + # `set -e` is not aborted by the false test in `warn` mode. + if [ "$(spv_mode)" = "require" ]; then + return 1 + fi + return 0 +} + +# Emit the xcframework paths a root refers to. A root is either an +# `*.xcframework` itself or a directory containing some. +spv_each() { + case "$1" in + *.xcframework) + if [ -d "$1" ]; then printf '%s\n' "$1"; fi + ;; + *) + for _spv_x in "$1"/*.xcframework; do + if [ -d "$_spv_x" ]; then printf '%s\n' "$_spv_x"; fi + done + ;; + esac + return 0 +} + +# Emit the per-slice .framework bundles inside an xcframework. +# +# Globbed rather than derived from the xcframework's name: stage_spm.sh stages +# Python.xcframework as Python-.xcframework, and a name-keyed lookup +# would quietly find nothing there — turning "this slice is unsigned" into "there +# was nothing to check". +spv_slice_frameworks() { + for _spv_slice in "$1"/*/; do + [ -d "$_spv_slice" ] || continue + for _spv_fw in "$_spv_slice"*.framework; do + if [ -d "$_spv_fw" ]; then printf '%s\n' "$_spv_fw"; fi + done + done + return 0 +} + +# Assert one signed bundle carries a usable provider signature: verifiable, not +# ad-hoc, securely timestamped, and (when configured) the expected team. +spv_assert_signature() { + _spv_t=$1 + + # `codesign -dv` rather than probing for a _CodeSignature directory: a + # versioned macOS bundle keeps it at Versions//_CodeSignature and the + # version directory is not always "A" (CPython uses e.g. Versions/3.14). + if ! codesign -dv "$_spv_t" >/dev/null 2>&1; then + spv_bad "$_spv_t: not signed at all;" \ + "its IPA receipt will report signed = false" || return 1 + return 0 + fi + + if ! _spv_out=$(codesign --verify --strict --verbose=4 "$_spv_t" 2>&1); then + spv_bad "$_spv_t: provider signature does not verify: $_spv_out" || return 1 + return 0 + fi + + _spv_info=$(codesign -dvvv "$_spv_t" 2>&1) || _spv_info="" + + if printf '%s\n' "$_spv_info" | grep -q '^Signature=adhoc'; then + spv_bad "$_spv_t: ad-hoc signature, not a provider signature" || return 1 + return 0 + fi + + if ! printf '%s\n' "$_spv_info" | grep -q '^Timestamp='; then + spv_bad "$_spv_t: no secure timestamp;" \ + "its IPA receipt will report isSecureTimestamp = false" || return 1 + return 0 + fi + + if [ -n "${SERIOUS_PYTHON_EXPECTED_TEAM_ID:-}" ]; then + _spv_team=$(printf '%s\n' "$_spv_info" | sed -n 's/^TeamIdentifier=//p' | head -1) + if [ "$_spv_team" != "$SERIOUS_PYTHON_EXPECTED_TEAM_ID" ]; then + spv_bad "$_spv_t: TeamIdentifier '$_spv_team' !=" \ + "expected '$SERIOUS_PYTHON_EXPECTED_TEAM_ID'" || return 1 + return 0 + fi + fi + return 0 +} + +# Verify the provider signature on every xcframework under the given roots — +# BOTH the outer bundle and each slice's inner .framework. +# +# Both layers matter. An XCFramework whose outer bundle is signed but whose inner +# frameworks are not produces an IPA receipt reading `signed = true` but +# `isSecureTimestamp = false`; every slice of an XCFramework Apple's App Store +# scan accepts is signed in its own right. Checking only the outer seal would let +# that regression reach a submission unnoticed, which is exactly how it reached +# one before. +# +# Roots must name PROVIDER artifacts only. Frameworks this plugin generates from +# the app's own site-packages are built locally, carry an ad-hoc signature, and +# are correctly excluded — requiring a provider identity on them would be wrong. +spv_verify_provider() { + _spv_mode=$(spv_mode) + if [ "$_spv_mode" = "off" ]; then + return 0 + fi + + _spv_status=0 + _spv_count=0 + _spv_slices=0 + _spv_problems=0 + for _spv_root in "$@"; do + [ -e "$_spv_root" ] || continue + for _spv_xcf in $(spv_each "$_spv_root"); do + _spv_count=$((_spv_count + 1)) + + # Outer bundle. Checked by file rather than via codesign so an + # entirely unsigned XCFramework reports the useful message. + if [ ! -f "$_spv_xcf/_CodeSignature/CodeResources" ]; then + spv_bad "$_spv_xcf: no provider signature (unsigned XCFramework);" \ + "its IPA receipt will report signed = false" || _spv_status=1 + continue + fi + spv_assert_signature "$_spv_xcf" || _spv_status=1 + + # Each slice's inner framework, in its own right. + for _spv_inner in $(spv_slice_frameworks "$_spv_xcf"); do + _spv_slices=$((_spv_slices + 1)) + spv_assert_signature "$_spv_inner" || _spv_status=1 + done + done + done + + if [ "$_spv_count" -eq 0 ]; then + # An empty tree must never read as "everything passed". + spv_bad "no *.xcframework found under: $*" || _spv_status=1 + elif [ "$_spv_problems" -gt 0 ]; then + spv_note "$_spv_problems signature problem(s) across $_spv_count provider" \ + "xcframework(s) and $_spv_slices slice framework(s); the resulting IPA" \ + "will report them as unsigned or without a secure timestamp" + else + spv_note "provider signatures verified on $_spv_count xcframework(s)" \ + "and $_spv_slices slice framework(s)" + fi + return $_spv_status +} + +# Record a digest manifest of everything under $2, keyed on paths relative to +# it, so the same manifest can be checked against a copy staged elsewhere. +# +# Symlinks are recorded by target, not followed: the macOS Python.framework is a +# versioned bundle whose symlinks are part of what codesign seals. +spv_manifest_record() { + _spv_out=$1 + _spv_root=$2 + [ -d "$_spv_root" ] || return 0 + + mkdir -p "$(dirname "$_spv_out")" + { + ( cd "$_spv_root" && find . -type f -exec shasum -a 256 {} + ) + ( cd "$_spv_root" && find . -type l -exec sh -c \ + 'for l; do printf "symlink:%s %s\n" "$(readlink "$l")" "$l"; done' _ {} + ) + } | LC_ALL=C sort > "$_spv_out" + spv_note "recorded $(wc -l < "$_spv_out" | tr -d ' ') digests for $_spv_root" +} + +# Check that every path recorded in $1 still exists under $2 with the same +# content. Extra files under $2 are ignored, so one manifest of the provider +# stdlib frameworks also validates the merged directory they are staged into +# alongside locally generated ones. +# +# Always fatal. Nothing in this repo is allowed to modify a provider bundle, so +# a mismatch is a bug here, not a property of whatever upstream release is +# pinned. +spv_manifest_check() { + _spv_manifest=$1 + _spv_root=$2 + _spv_label=${3:-$_spv_root} + + [ -f "$_spv_manifest" ] || { spv_note "no manifest at $_spv_manifest; skipping check"; return 0; } + [ -d "$_spv_root" ] || { echo "provider-verify: $_spv_root missing" >&2; return 1; } + + _spv_actual=$(mktemp) + { + ( cd "$_spv_root" && find . -type f -exec shasum -a 256 {} + ) + ( cd "$_spv_root" && find . -type l -exec sh -c \ + 'for l; do printf "symlink:%s %s\n" "$(readlink "$l")" "$l"; done' _ {} + ) + } | LC_ALL=C sort > "$_spv_actual" + + # Only the recorded lines have to be present; that is what makes this work + # against a directory holding additional, locally built frameworks. + _spv_missing=$(LC_ALL=C comm -23 "$_spv_manifest" "$_spv_actual") + rm -f "$_spv_actual" + + if [ -n "$_spv_missing" ]; then + echo "provider-verify: provider artifacts changed during staging ($_spv_label):" >&2 + printf '%s\n' "$_spv_missing" | head -20 >&2 || true + echo "provider-verify: editing anything inside a provider XCFramework destroys its" \ + "SDK-origin signature; stage it byte-for-byte instead" >&2 + return 1 + fi + spv_note "provider artifacts unchanged ($_spv_label)" +} diff --git a/src/serious_python_darwin/lib/serious_python_darwin.dart b/src/serious_python_darwin/lib/serious_python_darwin.dart index 4e8cb76f..9b8ef7c0 100644 --- a/src/serious_python_darwin/lib/serious_python_darwin.dart +++ b/src/serious_python_darwin/lib/serious_python_darwin.dart @@ -55,7 +55,6 @@ class SeriousPythonDarwin extends SeriousPythonPlatform { ]; final env = { - 'PYTHONINSPECT': '1', 'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONNOUSERSITE': '1', 'PYTHONUNBUFFERED': '1', diff --git a/src/serious_python_darwin/pubspec.yaml b/src/serious_python_darwin/pubspec.yaml index 9ef38137..a2d93ae1 100644 --- a/src/serious_python_darwin/pubspec.yaml +++ b/src/serious_python_darwin/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python_darwin description: iOS and macOS implementations of the serious_python plugin homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 environment: # The Swift Package Manager build path needs Flutter 3.44 / Dart 3.11 (the diff --git a/src/serious_python_linux/CHANGELOG.md b/src/serious_python_linux/CHANGELOG.md index 9b232ba5..099cf562 100644 --- a/src/serious_python_linux/CHANGELOG.md +++ b/src/serious_python_linux/CHANGELOG.md @@ -1,3 +1,48 @@ +## 4.5.1 + +* Re-pins the bundled python-build snapshot to **20260730** and `dart_bridge` to **1.7.1**. Both exist to sign the Apple XCFrameworks on both layers (see `serious_python_darwin` 4.5.1); no Python version moved from 20260729 (**3.12.13 / 3.13.14 / 3.14.6**), and nothing about Linux changed. + +## 4.5.0 + +* Re-pins the bundled python-build snapshot to **20260729** and `dart_bridge` to **1.7.0**. Both exist to provider-sign the Apple XCFrameworks (see `serious_python_darwin` 4.5.0). Nothing about Linux changed: no Python version moved from 20260727 (**3.12.13 / 3.13.14 / 3.14.6**), and `libdart_bridge-linux-*.so` is byte-identical to 1.6.1 — verified, not assumed. + +## 4.4.2 + +* Re-pins the bundled python-build snapshot to **20260727**. Its only functional change is to the iOS frameworks' `Info.plist` (see `serious_python_darwin` 4.4.2); no versions moved and the Linux runtime is unchanged from 20260726. + +## 4.4.1 + +* Re-pins the bundled python-build snapshot to **20260726**. Its only functional change is an iOS privacy-manifest fix (see `serious_python_darwin` 4.4.1); no versions moved and the Linux runtime is unchanged from 20260725. + +## 4.4.0 + +* Re-pins the bundled python-build snapshot to **20260725** (`dart_bridge` **1.5.1 → 1.6.1**). 1.6.1 is an Apple-only packaging change (see `serious_python_darwin` 4.4.0); the Linux runtime is byte-identical to 20260720. + +## 4.3.6 + +* Re-pins the bundled python-build snapshot to **20260720** (`dart_bridge` **1.5.0 → 1.5.1**). 1.5.1 is a Windows-only UTF-8 startup fix (see `serious_python_windows` 4.3.6); the Linux runtime is functionally unchanged from 20260719. + +## 4.3.4 + +* **Fix `ModuleNotFoundError: No module named '_pyrepl'` on Python 3.14.** `_pyrepl` was pruned from the bundled Linux stdlib as a dev-only / interactive-REPL module, but CPython 3.14's `pydoc` (and `pdb`) import it at module load — so any app importing `pydoc`/`pdb`/`pytest`, or a dependency that does (e.g. NLTK → `pydoc`), crashed at startup. `_pyrepl` is no longer pruned. Re-pins the bundled python-build snapshot to **20260719** (previously 20260714). ([flet-dev/serious-python#236](https://github.com/flet-dev/serious-python/issues/236)) + +## 4.3.3 + +* Remove a stray non-ASCII character (an em dash) from a source comment in `serious_python_linux_plugin.cc` — GCC reads UTF-8 sources by default so there was no build impact, but it mirrors the Windows fix (flet-dev/flet#6686). Version bump aligning with the `serious_python_*` 4.3.3 release; no Linux-affecting runtime changes. + +## 4.3.2 + +* Bump the bundled python-build snapshot to `20260714`; aligns with the `serious_python_*` 4.3.2 release. The release contains only Android/iOS runtime fixes — no Linux-affecting changes. + +## 4.3.1 + +* Version bump aligning with the `serious_python_*` 4.3.1 release. + +## 4.3.0 + +* Bump `dart_bridge` to **1.5.0** (python-build snapshot `20260708`): multiprocessing child-interception exports (`serious_python_is_mp_invocation` / `serious_python_main`), consumed by the flet build template's `main.cc` via `dlopen("libdart_bridge.so")`. Relevant on Linux since Python 3.14 made `forkserver` (which execs `sys.executable`) the default start method. See the `serious_python` 4.3.0 notes. +* `PYTHONINSPECT=1` is no longer set by any platform implementation. It had no effect on the embedded interpreter, but it leaked into the process environment where any *real* interpreter child (e.g. a serviced multiprocessing worker) would inherit it and hang in interactive mode after its command completed. + ## 4.2.1 * Bump the bundled python-build snapshot to `20260701`; aligns with the `serious_python_*` 4.2.1 release. The Linux runtimes are byte-identical to `20260630` (the release only rebuilds the iOS runtime). diff --git a/src/serious_python_linux/lib/serious_python_linux.dart b/src/serious_python_linux/lib/serious_python_linux.dart index 0c2a39cc..ea9c3147 100644 --- a/src/serious_python_linux/lib/serious_python_linux.dart +++ b/src/serious_python_linux/lib/serious_python_linux.dart @@ -48,7 +48,6 @@ class SeriousPythonLinux extends SeriousPythonPlatform { ]; final env = { - 'PYTHONINSPECT': '1', 'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONNOUSERSITE': '1', 'PYTHONUNBUFFERED': '1', diff --git a/src/serious_python_linux/linux/python_versions.properties b/src/serious_python_linux/linux/python_versions.properties index faf98a2e..0bdc3bba 100644 --- a/src/serious_python_linux/linux/python_versions.properties +++ b/src/serious_python_linux/linux/python_versions.properties @@ -1,8 +1,8 @@ # GENERATED by `dart run serious_python:gen_version_tables` from -# python-build manifest.json (release 20260701). Do not edit by hand. +# python-build manifest.json (release 20260730). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.4.1 -python_build_release_date=20260701 +dart_bridge_version=1.7.1 +python_build_release_date=20260730 3.12.full_version=3.12.13 3.12.android_abis=arm64-v8a,x86_64,armeabi-v7a 3.13.full_version=3.13.14 diff --git a/src/serious_python_linux/linux/serious_python_linux_plugin.cc b/src/serious_python_linux/linux/serious_python_linux_plugin.cc index e96f9191..63db046f 100644 --- a/src/serious_python_linux/linux/serious_python_linux_plugin.cc +++ b/src/serious_python_linux/linux/serious_python_linux_plugin.cc @@ -3,7 +3,7 @@ #include #include -// Plugin-registration shell only — all method calls return NotImplemented. +// Plugin-registration shell only - all method calls return NotImplemented. // Python lifecycle lives in libdart_bridge.so, invoked from Dart via FFI. #define SERIOUS_PYTHON_LINUX_PLUGIN(obj) \ diff --git a/src/serious_python_linux/pubspec.yaml b/src/serious_python_linux/pubspec.yaml index 526b5e34..17683d1d 100644 --- a/src/serious_python_linux/pubspec.yaml +++ b/src/serious_python_linux/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python_linux description: Linux implementations of the serious_python plugin homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 environment: sdk: '>=3.1.3 <4.0.0' diff --git a/src/serious_python_platform_interface/CHANGELOG.md b/src/serious_python_platform_interface/CHANGELOG.md index 0374f5fa..54336b7b 100644 --- a/src/serious_python_platform_interface/CHANGELOG.md +++ b/src/serious_python_platform_interface/CHANGELOG.md @@ -1,3 +1,47 @@ +## 4.5.1 + +* Version bump aligning with the `serious_python_*` 4.5.1 release. + +## 4.5.0 + +* Version bump aligning with the `serious_python_*` 4.5.0 release. + +## 4.4.2 + +* Version bump aligning with the `serious_python_*` 4.4.2 release. + +## 4.4.1 + +* Version bump aligning with the `serious_python_*` 4.4.1 release. + +## 4.4.0 + +* Version bump aligning with the `serious_python_*` 4.4.0 release. + +## 4.3.6 + +* Version bump aligning with the `serious_python_*` 4.3.6 release (Windows UTF-8 startup fix via `dart_bridge` 1.5.1). No interface changes. + +## 4.3.4 + +* Version bump aligning with the `serious_python_*` 4.3.4 release (desktop `_pyrepl` fix + Android `flet debug` cache fix). No interface changes. + +## 4.3.3 + +* Version bump aligning with the `serious_python_*` 4.3.3 release. + +## 4.3.2 + +* Version bump aligning with the `serious_python_*` 4.3.2 release. + +## 4.3.1 + +* Version bump aligning with the `serious_python_*` 4.3.1 release. + +## 4.3.0 + +* Version bump aligning with the `serious_python_*` 4.3.0 release. + ## 4.2.1 * Version bump aligning with the `serious_python_*` 4.2.1 release. diff --git a/src/serious_python_platform_interface/pubspec.yaml b/src/serious_python_platform_interface/pubspec.yaml index ea303247..dedef6ba 100644 --- a/src/serious_python_platform_interface/pubspec.yaml +++ b/src/serious_python_platform_interface/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python_platform_interface description: A common platform interface for the serious_python plugin. homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 environment: sdk: ">=3.0.0 <4.0.0" diff --git a/src/serious_python_windows/CHANGELOG.md b/src/serious_python_windows/CHANGELOG.md index efb47934..34f29de1 100644 --- a/src/serious_python_windows/CHANGELOG.md +++ b/src/serious_python_windows/CHANGELOG.md @@ -1,3 +1,49 @@ +## 4.5.1 + +* Re-pins the bundled python-build snapshot to **20260730** and `dart_bridge` to **1.7.1**. Both exist to sign the Apple XCFrameworks on both layers (see `serious_python_darwin` 4.5.1); no Python version moved from 20260729 (**3.12.13 / 3.13.14 / 3.14.6**), and nothing about Windows changed. + +## 4.5.0 + +* Re-pins the bundled python-build snapshot to **20260729** and `dart_bridge` to **1.7.0**. Both exist to provider-sign the Apple XCFrameworks (see `serious_python_darwin` 4.5.0); no Python version moved from 20260727 (**3.12.13 / 3.13.14 / 3.14.6**). `dart_bridge.dll` is rebuilt rather than byte-identical — MSVC stamps a build timestamp and PDB GUID into every DLL — but no Windows source changed between 1.6.1 and 1.7.0. + +## 4.4.2 + +* Re-pins the bundled python-build snapshot to **20260727**. Its only functional change is to the iOS frameworks' `Info.plist` (see `serious_python_darwin` 4.4.2); no versions moved and the Windows runtime is unchanged from 20260726. + +## 4.4.1 + +* Re-pins the bundled python-build snapshot to **20260726**. Its only functional change is an iOS privacy-manifest fix (see `serious_python_darwin` 4.4.1); no versions moved and the Windows runtime is unchanged from 20260725. + +## 4.4.0 + +* Re-pins the bundled python-build snapshot to **20260725** (`dart_bridge` **1.5.1 → 1.6.1**). 1.6.1 is an Apple-only packaging change (see `serious_python_darwin` 4.4.0); the Windows runtime is byte-identical to 20260720. + +## 4.3.6 + +* **Fix Windows startup when app paths or environment values contain non-ASCII characters.** Dart FFI strings arrive as UTF-8, but the Windows boundary passed them straight to the ANSI CRT (`_putenv_s`, `fopen`), corrupting paths/env values through the process code page. The runtime now converts UTF-8 → UTF-16 and uses the wide CRT APIs (`_wputenv_s`, `_wfopen_s`), enables Python UTF-8 mode before `Py_Initialize()`, sets `PYTHONUTF8=1` for multiprocessing helpers, and treats embedded env-setup failures as fatal. Delivered via `dart_bridge` **1.5.1** (bundled python-build snapshot re-pinned to **20260720**). ([flet-dev/flet#6641](https://github.com/flet-dev/flet/issues/6641)) + +## 4.3.4 + +* **Fix `ModuleNotFoundError: No module named '_pyrepl'` on Python 3.14.** `_pyrepl` was pruned from the bundled Windows stdlib as a dev-only / interactive-REPL module, but CPython 3.14's `pydoc` (and `pdb`) import it at module load — so any app importing `pydoc`/`pdb`/`pytest`, or a dependency that does (e.g. NLTK → `pydoc`), crashed at startup. `_pyrepl` is no longer pruned. Re-pins the bundled python-build snapshot to **20260719** (previously 20260714). ([flet-dev/serious-python#236](https://github.com/flet-dev/serious-python/issues/236)) + +## 4.3.3 + +* **Fix `flet build windows` failing on non-UTF-8 system locales** with `warning C4819` escalated to `error C2220` while compiling `serious_python_windows_plugin.cpp` (flet-dev/flet#6686). A source comment contained a non-ASCII character (an em dash); on a system whose code page isn't UTF-8 — e.g. code page **936/GBK** on Simplified-Chinese Windows — MSVC decodes the UTF-8 source as GBK, can't represent the byte sequence (C4819), and the Flutter template's `/WX` (warnings-as-errors) turns it into a fatal C2220. The character is removed, and the plugin now compiles with `/utf-8` so any future non-ASCII source byte is read correctly regardless of the build machine's code page. Bundled Python and `dart_bridge` versions are unchanged from 4.3.2. + +## 4.3.2 + +* Bump the bundled python-build snapshot to `20260714`; aligns with the `serious_python_*` 4.3.2 release. The release contains only Android/iOS runtime fixes — no Windows-affecting changes. + +## 4.3.1 + +* Version bump aligning with the `serious_python_*` 4.3.1 release. + +## 4.3.0 + +* Bump `dart_bridge` to **1.5.0** (python-build snapshot `20260708`): multiprocessing child-interception exports, including the Windows wide-char variants `serious_python_is_mp_invocation_w` / `serious_python_main_w` (→ `Py_Main`) consumed by the flet build template's `wWinMain`. See the `serious_python` 4.3.0 notes. +* `PYTHONINSPECT=1` is no longer set by any platform implementation. It had no effect on the embedded interpreter, but it leaked into the process environment where any *real* interpreter child (e.g. a serviced multiprocessing worker) would inherit it and hang in interactive mode after its command completed. +* **Fix `flet build windows` failing with `file INSTALL cannot find "C:/WINDOWS/System32/vcruntime140_1.dll"`** for users who have VS Build Tools installed rather than full Visual Studio. The plugin harvests the CRT runtime DLLs (`msvcp140.dll` / `vcruntime140.dll` / `vcruntime140_1.dll`) from `%WINDIR%\System32`, but Flutter drives the CMake install step with the **32-bit** `cmake.exe` bundled in VS Build Tools. Under WOW64 file-system redirection that process sees `System32` transparently rewritten to `SysWOW64`, which holds the x86 CRT and doesn't contain `vcruntime140_1.dll` at all — so the x64 build copied wrong-arch DLLs and then failed. The CRT directory is now resolved via the `Sysnative` pseudo-folder (visible only to 32-bit processes, mapping back to the real 64-bit `System32`) when present, falling back to `System32` for native 64-bit cmake. See flet-dev/flet#6436. + ## 4.2.1 * Bump the bundled python-build snapshot to `20260701`; aligns with the `serious_python_*` 4.2.1 release. The Windows runtimes are byte-identical to `20260630` (the release only rebuilds the iOS runtime). diff --git a/src/serious_python_windows/lib/serious_python_windows.dart b/src/serious_python_windows/lib/serious_python_windows.dart index de0a6a8a..5c6861ba 100644 --- a/src/serious_python_windows/lib/serious_python_windows.dart +++ b/src/serious_python_windows/lib/serious_python_windows.dart @@ -46,7 +46,6 @@ class SeriousPythonWindows extends SeriousPythonPlatform { ]; final env = { - 'PYTHONINSPECT': '1', 'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONNOUSERSITE': '1', 'PYTHONUNBUFFERED': '1', diff --git a/src/serious_python_windows/pubspec.yaml b/src/serious_python_windows/pubspec.yaml index 03a0bf20..20432051 100644 --- a/src/serious_python_windows/pubspec.yaml +++ b/src/serious_python_windows/pubspec.yaml @@ -2,7 +2,7 @@ name: serious_python_windows description: Windows implementations of the serious_python plugin homepage: https://flet.dev repository: https://github.com/flet-dev/serious-python -version: 4.2.1 +version: 4.5.1 environment: sdk: '>=3.1.3 <4.0.0' diff --git a/src/serious_python_windows/windows/CMakeLists.txt b/src/serious_python_windows/windows/CMakeLists.txt index 573388c6..ce925f39 100644 --- a/src/serious_python_windows/windows/CMakeLists.txt +++ b/src/serious_python_windows/windows/CMakeLists.txt @@ -172,6 +172,14 @@ add_library(${PLUGIN_NAME} SHARED apply_standard_settings(${PLUGIN_NAME}) target_compile_features(${PLUGIN_NAME} PRIVATE cxx_std_20) +# Force MSVC to read source files as UTF-8 regardless of the build machine's +# system code page. Without this, a non-ASCII byte anywhere in a source file +# triggers warning C4819 on non-UTF-8 locales (e.g. code page 936/GBK on +# Simplified-Chinese Windows), which the Flutter template's /WX +# (warnings-as-errors) escalates to a fatal C2220 build error +# (flet-dev/flet#6686). +target_compile_options(${PLUGIN_NAME} PRIVATE "/utf-8") + # Symbols are hidden by default to reduce the chance of accidental conflicts # between plugins. This should not be removed; any symbols that should be # exported should be explicitly exported with the FLUTTER_PLUGIN_EXPORT macro. @@ -190,13 +198,31 @@ target_link_libraries(${PLUGIN_NAME} PRIVATE flutter flutter_wrapper_plugin) # This list could contain prebuilt libraries, or libraries created by an # external build triggered from this build file. string(REPLACE "\\" "/" SERIOUS_PYTHON_WINDIR "$ENV{WINDIR}") + +# CRT runtime DLLs (msvcp140 / vcruntime140 / vcruntime140_1) are harvested from +# the OS's 64-bit System32. Flutter may drive the CMake install step with the +# 32-bit cmake.exe bundled in VS Build Tools (users without full Visual Studio). +# Under WOW64 file-system redirection a 32-bit process sees C:\Windows\System32 +# transparently rewritten to SysWOW64, which holds the x86 CRT and doesn't +# contain vcruntime140_1.dll at all — so the x64 build copies wrong-arch DLLs and +# then fails with "cannot find vcruntime140_1.dll" (flet-dev/flet#6436). +# "Sysnative" is the documented pseudo-folder that maps a 32-bit process back to +# the real 64-bit System32; it's visible only to 32-bit processes, so a native +# 64-bit cmake has no Sysnative and keeps using System32. Configure- and +# install-time cmake are the same binary under Flutter, so the path resolved here +# stays valid when cmake_install.cmake later runs. +set(_sp_crt_dir "${SERIOUS_PYTHON_WINDIR}/System32") +if(EXISTS "${SERIOUS_PYTHON_WINDIR}/Sysnative") + set(_sp_crt_dir "${SERIOUS_PYTHON_WINDIR}/Sysnative") +endif() + set(serious_python_windows_bundled_libraries "${PYTHON_PACKAGE}/python${PYTHON_VERSION_NODOT}$<$:_d>.dll" "${PYTHON_PACKAGE}/python3$<$:_d>.dll" "$,${DART_BRIDGE_DEBUG_DLL},${DART_BRIDGE_RELEASE_DLL}>" - "${SERIOUS_PYTHON_WINDIR}/System32/msvcp140.dll" - "${SERIOUS_PYTHON_WINDIR}/System32/vcruntime140.dll" - "${SERIOUS_PYTHON_WINDIR}/System32/vcruntime140_1.dll" + "${_sp_crt_dir}/msvcp140.dll" + "${_sp_crt_dir}/vcruntime140.dll" + "${_sp_crt_dir}/vcruntime140_1.dll" PARENT_SCOPE ) diff --git a/src/serious_python_windows/windows/python_versions.properties b/src/serious_python_windows/windows/python_versions.properties index faf98a2e..0bdc3bba 100644 --- a/src/serious_python_windows/windows/python_versions.properties +++ b/src/serious_python_windows/windows/python_versions.properties @@ -1,8 +1,8 @@ # GENERATED by `dart run serious_python:gen_version_tables` from -# python-build manifest.json (release 20260701). Do not edit by hand. +# python-build manifest.json (release 20260730). Do not edit by hand. default_python_version=3.14 -dart_bridge_version=1.4.1 -python_build_release_date=20260701 +dart_bridge_version=1.7.1 +python_build_release_date=20260730 3.12.full_version=3.12.13 3.12.android_abis=arm64-v8a,x86_64,armeabi-v7a 3.13.full_version=3.13.14 diff --git a/src/serious_python_windows/windows/serious_python_windows_plugin.cpp b/src/serious_python_windows/windows/serious_python_windows_plugin.cpp index 62f9a489..ddc2447b 100644 --- a/src/serious_python_windows/windows/serious_python_windows_plugin.cpp +++ b/src/serious_python_windows/windows/serious_python_windows_plugin.cpp @@ -36,7 +36,7 @@ namespace serious_python_windows SeriousPythonWindowsPlugin::~SeriousPythonWindowsPlugin() {} - // Plugin-registration shell only — all method calls return NotImplemented. + // Plugin-registration shell only - all method calls return NotImplemented. // Python lifecycle lives in dart_bridge[_d].dll, invoked from Dart via FFI. void SeriousPythonWindowsPlugin::HandleMethodCall( const flutter::MethodCall &method_call,