From 899ffd1ad33fd7ea67703a5335766196870815e6 Mon Sep 17 00:00:00 2001 From: Peng Xiao Date: Wed, 18 Jun 2025 13:57:01 +0800 Subject: [PATCH] feat(native): windows audio monitoring & recording (#12615) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix AF-2692 ## Summary by CodeRabbit - **New Features** - Added comprehensive Windows support for audio and application capture, including real-time microphone usage detection, combined microphone and system audio recording, and application state monitoring. - The "meetings" setting is now enabled on Windows as well as macOS. - Conditional UI styling and attributes introduced for Windows environments in the Electron renderer. - **Bug Fixes** - Enhanced file path handling and validation for Windows in Electron file requests. - **Refactor** - Unified application info handling across platforms by consolidating types into a single `ApplicationInfo` structure. - Updated native module APIs by removing deprecated types, refining method signatures, and improving error messages. - Streamlined audio tapping APIs to use process IDs and consistent callback types. - **Documentation** - Added detailed documentation for the Windows-specific audio recording and microphone listener modules. - **Chores** - Updated development dependencies in multiple packages. - Reorganized and added platform-specific dependencies and configuration for Windows support. #### PR Dependency Tree * **PR #12615** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) --------- Co-authored-by: LongYinan --- Cargo.lock | 332 ++++++++- Cargo.toml | 15 + packages/backend/native/package.json | 2 +- packages/common/y-octo/node/package.json | 2 +- .../electron-renderer/src/popup/app.css.ts | 6 + .../apps/electron-renderer/src/popup/app.tsx | 2 +- .../apps/electron/src/main/protocol.ts | 21 +- .../electron/src/main/recording/feature.ts | 89 ++- .../apps/electron/src/main/recording/types.ts | 4 +- .../src/main/windows-manager/popup.ts | 3 + .../dialogs/setting/general-setting/index.tsx | 5 +- .../general-setting/meetings/index.tsx | 47 +- .../media-capture-playground/server/main.ts | 218 ++++-- packages/frontend/native/index.d.ts | 133 ++-- packages/frontend/native/index.js | 23 +- .../frontend/native/media_capture/Cargo.toml | 16 +- .../frontend/native/media_capture/src/lib.rs | 6 + .../src/macos/screen_capture_kit.rs | 368 +++++----- .../media_capture/src/macos/tap_audio.rs | 23 +- .../media_capture/src/windows/README.md | 86 +++ .../src/windows/audio_capture.rs | 400 +++++++++++ .../native/media_capture/src/windows/error.rs | 39 ++ .../src/windows/microphone_listener.rs | 645 ++++++++++++++++++ .../native/media_capture/src/windows/mod.rs | 17 + .../src/windows/screen_capture_kit.rs | 449 ++++++++++++ packages/frontend/native/package.json | 2 +- yarn.lock | 14 +- 27 files changed, 2509 insertions(+), 458 deletions(-) create mode 100644 packages/frontend/native/media_capture/src/windows/README.md create mode 100644 packages/frontend/native/media_capture/src/windows/audio_capture.rs create mode 100644 packages/frontend/native/media_capture/src/windows/error.rs create mode 100644 packages/frontend/native/media_capture/src/windows/microphone_listener.rs create mode 100644 packages/frontend/native/media_capture/src/windows/mod.rs create mode 100644 packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs diff --git a/Cargo.lock b/Cargo.lock index 971a8646cd..eb22962147 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,8 +77,10 @@ version = "0.0.0" dependencies = [ "block2", "core-foundation", - "coreaudio-rs", + "coreaudio-rs 0.12.1", + "cpal", "criterion2", + "crossbeam-channel", "dispatch2", "libc", "napi", @@ -91,6 +93,8 @@ dependencies = [ "symphonia", "thiserror 2.0.12", "uuid", + "windows 0.61.1", + "windows-core 0.61.2", ] [[package]] @@ -213,6 +217,28 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "alsa" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +dependencies = [ + "alsa-sys", + "bitflags 2.9.1", + "cfg-if", + "libc", +] + +[[package]] +name = "alsa-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -695,9 +721,17 @@ version = "1.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7" dependencies = [ + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -853,6 +887,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -979,6 +1023,17 @@ dependencies = [ "libm", ] +[[package]] +name = "coreaudio-rs" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace" +dependencies = [ + "bitflags 1.3.2", + "core-foundation-sys", + "coreaudio-sys", +] + [[package]] name = "coreaudio-rs" version = "0.12.1" @@ -999,6 +1054,29 @@ dependencies = [ "bindgen", ] +[[package]] +name = "cpal" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779" +dependencies = [ + "alsa", + "core-foundation-sys", + "coreaudio-rs 0.11.3", + "dasp_sample", + "jni", + "js-sys", + "libc", + "mach2", + "ndk", + "ndk-context", + "oboe", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows 0.54.0", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1085,6 +1163,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1165,6 +1252,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "dasp_sample" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f" + [[package]] name = "der" version = "0.7.10" @@ -1820,7 +1913,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.57.0", ] [[package]] @@ -2082,6 +2175,38 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47f142fe24a9c9944451e8349de0a56af5f3e7226dc46f3ed4d4ecc0b85af75e" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -2159,7 +2284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -2264,6 +2389,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -2370,9 +2504,9 @@ dependencies = [ [[package]] name = "napi" -version = "3.0.0-beta.3" +version = "3.0.0-beta.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5c343e6e1fb57bf3ea3386638c4affb394ee932708128840a56aaac3d6a8ab" +checksum = "c502f122fc89e92c6222810b3144411c6f945da5aa3b713ddfad3bdcae7c9bb4" dependencies = [ "anyhow", "bitflags 2.9.1", @@ -2380,21 +2514,23 @@ dependencies = [ "ctor", "napi-build", "napi-sys", + "nohash-hasher", + "rustc-hash 2.1.1", "serde", "tokio", ] [[package]] name = "napi-build" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03acbfa4f156a32188bfa09b86dc11a431b5725253fc1fc6f6df5bed273382c4" +checksum = "44e0e3177307063d3e7e55b7dd7b648cca9d7f46daa35422c0d98cc2bf48c2c1" [[package]] name = "napi-derive" -version = "3.0.0-beta.3" +version = "3.0.0-beta.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d23065ee795a4b1a8755fdf4a39c2a229679f01f923a8feea33f045d6d96cb" +checksum = "fcf1e732a67e934b069d6d527251d6288753a36840572abe132a7aed9e77f0bc" dependencies = [ "convert_case 0.8.0", "ctor", @@ -2406,9 +2542,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "2.0.0-beta.3" +version = "2.0.0-beta.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348aaac2c51b5d11cf90cf7670b470c7f4d1607d15c338efd4d3db361003e4f5" +checksum = "462b775ba74791c98989fadc46c4bb2ec53016427be4d420d31c4bbaab34b308" dependencies = [ "convert_case 0.8.0", "proc-macro2", @@ -2419,13 +2555,42 @@ dependencies = [ [[package]] name = "napi-sys" -version = "3.0.0-alpha.2" +version = "3.0.0-alpha.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b443b980b2258dbaa31b99115e74da6c0866e537278309d566b4672a2f8df516" +checksum = "c4401c63f866b42d673a8b213d5662c84a0701b0f6c3acff7e2b9fc439f1675d" dependencies = [ "libloading", ] +[[package]] +name = "ndk" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7" +dependencies = [ + "bitflags 2.9.1", + "jni-sys", + "log", + "ndk-sys", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + +[[package]] +name = "ndk-sys" +version = "0.5.0+25.2.9519653" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" +dependencies = [ + "jni-sys", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2444,6 +2609,12 @@ dependencies = [ "libc", ] +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -2510,6 +2681,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2628,6 +2810,29 @@ dependencies = [ "memchr", ] +[[package]] +name = "oboe" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb" +dependencies = [ + "jni", + "ndk", + "ndk-context", + "num-derive", + "num-traits", + "oboe-sys", +] + +[[package]] +name = "oboe-sys" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d" +dependencies = [ + "cc", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -5003,6 +5208,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.100" @@ -5116,7 +5334,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -5125,6 +5343,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core 0.54.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows" version = "0.57.0" @@ -5157,6 +5385,16 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.57.0" @@ -5280,6 +5518,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5307,6 +5554,21 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5347,6 +5609,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5359,6 +5627,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5371,6 +5645,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5389,6 +5669,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5401,6 +5687,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5413,6 +5705,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5425,6 +5723,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index 6c2495aa3c..424c03e22e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,12 +28,14 @@ base64-simd = "0.8" bitvec = "1.0" block2 = "0.6" byteorder = "1.5" +cpal = "0.15" chrono = "0.4" clap = { version = "4.4", features = ["derive"] } core-foundation = "0.10" coreaudio-rs = "0.12" criterion = { version = "0.5", features = ["html_reports"] } criterion2 = { version = "3", default-features = false } +crossbeam-channel = "0.5" dispatch2 = "0.3" docx-parser = { git = "https://github.com/toeverything/docx-parser" } dotenvy = "0.15" @@ -97,6 +99,19 @@ uniffi = "0.29" url = { version = "2.5" } uuid = "1.8" v_htmlescape = "0.15" +windows = { version = "0.61", features = [ + "Win32_Devices_FunctionDiscovery", + "Win32_UI_Shell_PropertiesSystem", + "Win32_Media_Audio", + "Win32_System_Variant", + "Win32_System_Com_StructuredStorage", + "Win32_System_Threading", + "Win32_System_ProcessStatus", + "Win32_Foundation", + "Win32_System_Com", + "Win32_System_Diagnostics_ToolHelp", +] } +windows-core = { version = "0.61" } y-octo = { path = "./packages/common/y-octo/core" } y-sync = { version = "0.4" } yrs = "0.23.0" diff --git a/packages/backend/native/package.json b/packages/backend/native/package.json index 791bd0fc2c..672962344c 100644 --- a/packages/backend/native/package.json +++ b/packages/backend/native/package.json @@ -32,7 +32,7 @@ "build:debug": "napi build" }, "devDependencies": { - "@napi-rs/cli": "3.0.0-alpha.81", + "@napi-rs/cli": "3.0.0-alpha.89", "lib0": "^0.2.99", "tiktoken": "^1.0.17", "tinybench": "^4.0.0", diff --git a/packages/common/y-octo/node/package.json b/packages/common/y-octo/node/package.json index 3b4523aef6..b9474a985d 100644 --- a/packages/common/y-octo/node/package.json +++ b/packages/common/y-octo/node/package.json @@ -21,7 +21,7 @@ }, "license": "MIT", "devDependencies": { - "@napi-rs/cli": "3.0.0-alpha.81", + "@napi-rs/cli": "3.0.0-alpha.89", "@types/node": "^22.14.1", "@types/prompts": "^2.4.9", "c8": "^10.1.3", diff --git a/packages/frontend/apps/electron-renderer/src/popup/app.css.ts b/packages/frontend/apps/electron-renderer/src/popup/app.css.ts index 7667c08180..33bb89ca87 100644 --- a/packages/frontend/apps/electron-renderer/src/popup/app.css.ts +++ b/packages/frontend/apps/electron-renderer/src/popup/app.css.ts @@ -1,3 +1,4 @@ +import { cssVarV2 } from '@toeverything/theme/v2'; import { globalStyle, style } from '@vanilla-extract/css'; globalStyle('html', { @@ -13,4 +14,9 @@ export const root = style({ width: '100%', height: '100%', userSelect: 'none', + selectors: { + '&[data-is-windows]': { + backgroundColor: cssVarV2('layer/background/primary'), + }, + }, }); diff --git a/packages/frontend/apps/electron-renderer/src/popup/app.tsx b/packages/frontend/apps/electron-renderer/src/popup/app.tsx index 244f34bbf8..28c16001ca 100644 --- a/packages/frontend/apps/electron-renderer/src/popup/app.tsx +++ b/packages/frontend/apps/electron-renderer/src/popup/app.tsx @@ -25,7 +25,7 @@ export function App() { -
+
{mode === 'recording' && }
diff --git a/packages/frontend/apps/electron/src/main/protocol.ts b/packages/frontend/apps/electron/src/main/protocol.ts index cf5db6b57b..edc1749157 100644 --- a/packages/frontend/apps/electron/src/main/protocol.ts +++ b/packages/frontend/apps/electron/src/main/protocol.ts @@ -1,9 +1,10 @@ -import { join } from 'node:path'; +import path, { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; import { app, net, protocol, session } from 'electron'; import cookieParser from 'set-cookie-parser'; -import { resourcesPath } from '../shared/utils'; +import { isWindows, resourcesPath } from '../shared/utils'; import { anotherHost, mainHost } from './constants'; import { logger } from './logger'; @@ -77,17 +78,23 @@ async function handleFileRequest(request: Request) { } } else { filepath = decodeURIComponent(urlObject.pathname); + // on windows, the path could be start with '/' + if (isWindows()) { + filepath = path.resolve(filepath.replace(/^\//, '')); + } // security check if the filepath is within app.getPath('sessionData') - const sessionDataPath = app.getPath('sessionData'); - const tempPath = app.getPath('temp'); + const sessionDataPath = path + .resolve(app.getPath('sessionData')) + .toLowerCase(); + const tempPath = path.resolve(app.getPath('temp')).toLowerCase(); if ( - !filepath.startsWith(sessionDataPath) && - !filepath.startsWith(tempPath) + !filepath.toLowerCase().startsWith(sessionDataPath) && + !filepath.toLowerCase().startsWith(tempPath) ) { throw new Error('Invalid filepath'); } } - return net.fetch('file://' + filepath, clonedRequest); + return net.fetch(pathToFileURL(filepath).toString(), clonedRequest); } export function registerProtocol() { diff --git a/packages/frontend/apps/electron/src/main/recording/feature.ts b/packages/frontend/apps/electron/src/main/recording/feature.ts index 6d4fbdd1f0..adecc83529 100644 --- a/packages/frontend/apps/electron/src/main/recording/feature.ts +++ b/packages/frontend/apps/electron/src/main/recording/feature.ts @@ -1,10 +1,11 @@ /* oxlint-disable no-var-requires */ import { execSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import fsp from 'node:fs/promises'; import path from 'node:path'; // Should not load @affine/native for unsupported platforms -import type { ShareableContent } from '@affine/native'; +import type { ShareableContent as ShareableContentType } from '@affine/native'; import { app, systemPreferences } from 'electron'; import fs from 'fs-extra'; import { debounce } from 'lodash-es'; @@ -19,7 +20,7 @@ import { } from 'rxjs'; import { filter, map, shareReplay } from 'rxjs/operators'; -import { isMacOS, shallowEqual } from '../../shared/utils'; +import { isMacOS, isWindows, shallowEqual } from '../../shared/utils'; import { beforeAppQuit } from '../cleanup'; import { logger } from '../logger'; import { @@ -64,7 +65,7 @@ export const SAVED_RECORDINGS_DIR = path.join( 'recordings' ); -let shareableContent: ShareableContent | null = null; +let shareableContent: ShareableContentType | null = null; function cleanup() { shareableContent = null; @@ -95,8 +96,10 @@ const recordings = new Map(); export const recordingStatus$ = recordingStateMachine.status$; function createAppGroup(processGroupId: number): AppGroupInfo | undefined { - const groupProcess = - shareableContent?.applicationWithProcessId(processGroupId); + // MUST require dynamically to avoid loading @affine/native for unsupported platforms + const SC: typeof ShareableContentType = + require('@affine/native').ShareableContent; + const groupProcess = SC?.applicationWithProcessId(processGroupId); if (!groupProcess) { return; } @@ -239,15 +242,30 @@ function setupNewRunningAppGroup() { ); } +function getSanitizedAppId(bundleIdentifier?: string) { + if (!bundleIdentifier) { + return 'unknown'; + } + + return isWindows() + ? createHash('sha256') + .update(bundleIdentifier) + .digest('hex') + .substring(0, 8) + : bundleIdentifier; +} + export function createRecording(status: RecordingStatus) { let recording = recordings.get(status.id); if (recording) { return recording; } + const appId = getSanitizedAppId(status.appGroup?.bundleIdentifier); + const bufferedFilePath = path.join( SAVED_RECORDINGS_DIR, - `${status.appGroup?.bundleIdentifier ?? 'unknown'}-${status.id}-${status.startTime}.raw` + `${appId}-${status.id}-${status.startTime}.raw` ); fs.ensureDirSync(SAVED_RECORDINGS_DIR); @@ -273,11 +291,12 @@ export function createRecording(status: RecordingStatus) { } // MUST require dynamically to avoid loading @affine/native for unsupported platforms - const ShareableContent = require('@affine/native').ShareableContent; + const SC: typeof ShareableContentType = + require('@affine/native').ShareableContent; const stream = status.app - ? status.app.rawInstance.tapAudio(tapAudioSamples) - : ShareableContent.tapGlobalAudio(null, tapAudioSamples); + ? SC.tapAudio(status.app.processId, tapAudioSamples) + : SC.tapGlobalAudio(null, tapAudioSamples); recording = { id: status.id, @@ -379,15 +398,24 @@ function getAllApps(): TappableAppInfo[] { if (!shareableContent) { return []; } - const apps = shareableContent.applications().map(app => { + + // MUST require dynamically to avoid loading @affine/native for unsupported platforms + const { ShareableContent } = require('@affine/native') as { + ShareableContent: typeof ShareableContentType; + }; + + const apps = ShareableContent.applications().map(app => { try { + // Check if this process is actively using microphone/audio + const isRunning = ShareableContent.isUsingMicrophone(app.processId); + return { - rawInstance: app, + info: app, processId: app.processId, processGroupId: app.processGroupId, bundleIdentifier: app.bundleIdentifier, name: app.name, - isRunning: app.isRunning, + isRunning, }; } catch (error) { logger.error('failed to get app info', error); @@ -441,15 +469,15 @@ function setupMediaListeners() { apps.forEach(app => { try { - const tappableApp = app.rawInstance; + const applicationInfo = app.info; _appStateSubscribers.push( - ShareableContent.onAppStateChanged(tappableApp, () => { + ShareableContent.onAppStateChanged(applicationInfo, () => { updateApplicationsPing$.next(Date.now()); }) ); } catch (error) { logger.error( - `Failed to convert app ${app.name} to TappableApplication`, + `Failed to set up app state listener for ${app.name}`, error ); } @@ -668,15 +696,18 @@ export async function readyRecording(id: number, buffer: Buffer) { return; } - const filepath = path.join( - SAVED_RECORDINGS_DIR, - `${recordingStatus.appGroup?.bundleIdentifier ?? 'unknown'}-${recordingStatus.id}-${recordingStatus.startTime}.opus` - ); + const rawFilePath = String(recording.file.path); + + const filepath = rawFilePath.replace('.raw', '.opus'); + + if (!filepath) { + logger.error(`readyRecording: Recording ${id} has no filepath`); + return; + } await fs.writeFile(filepath, buffer); // can safely remove the raw file now - const rawFilePath = recording.file.path; logger.info('remove raw file', rawFilePath); if (rawFilePath) { try { @@ -768,14 +799,24 @@ export const getMacOSVersion = () => { // check if the system is MacOS and the version is >= 14.2 export const checkRecordingAvailable = () => { - if (!isMacOS()) { - return false; + if (isMacOS()) { + const version = getMacOSVersion(); + return (version.major === 14 && version.minor >= 2) || version.major > 14; } - const version = getMacOSVersion(); - return (version.major === 14 && version.minor >= 2) || version.major > 14; + if (isWindows()) { + return true; + } + return false; }; export const checkMeetingPermissions = () => { + if (isWindows()) { + return { + screen: true, + microphone: true, + }; + } + if (!isMacOS()) { return undefined; } diff --git a/packages/frontend/apps/electron/src/main/recording/types.ts b/packages/frontend/apps/electron/src/main/recording/types.ts index b1d504acc9..cd6f3afc48 100644 --- a/packages/frontend/apps/electron/src/main/recording/types.ts +++ b/packages/frontend/apps/electron/src/main/recording/types.ts @@ -1,9 +1,9 @@ import type { WriteStream } from 'node:fs'; -import type { AudioCaptureSession, TappableApplication } from '@affine/native'; +import type { ApplicationInfo, AudioCaptureSession } from '@affine/native'; export interface TappableAppInfo { - rawInstance: TappableApplication; + info: ApplicationInfo; isRunning: boolean; processId: number; processGroupId: number; diff --git a/packages/frontend/apps/electron/src/main/windows-manager/popup.ts b/packages/frontend/apps/electron/src/main/windows-manager/popup.ts index c724a397be..7e79f94aee 100644 --- a/packages/frontend/apps/electron/src/main/windows-manager/popup.ts +++ b/packages/frontend/apps/electron/src/main/windows-manager/popup.ts @@ -77,6 +77,7 @@ abstract class PopupWindow { closable: false, alwaysOnTop: true, hiddenInMissionControl: true, + skipTaskbar: true, movable: false, titleBarStyle: 'hidden', show: false, // hide by default, @@ -243,6 +244,8 @@ export class PopupManager { return new NotificationPopupWindow() as PopupWindowTypeMap[T]; case 'recording': return new RecordingPopupWindow() as PopupWindowTypeMap[T]; + default: + throw new Error(`Unknown popup type: ${type}`); } })(); diff --git a/packages/frontend/core/src/desktop/dialogs/setting/general-setting/index.tsx b/packages/frontend/core/src/desktop/dialogs/setting/general-setting/index.tsx index 33547ac776..33436fa49f 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/general-setting/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/general-setting/index.tsx @@ -95,7 +95,10 @@ export const useGeneralSettingList = (): GeneralSettingList => { }); } - if (environment.isMacOs && BUILD_CONFIG.isElectron) { + if ( + (environment.isMacOs || environment.isWindows) && + BUILD_CONFIG.isElectron + ) { settings.push({ key: 'meetings', title: t['com.affine.settings.meetings'](), diff --git a/packages/frontend/core/src/desktop/dialogs/setting/general-setting/meetings/index.tsx b/packages/frontend/core/src/desktop/dialogs/setting/general-setting/meetings/index.tsx index d6ba6fa6a5..62a54a1c08 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/general-setting/meetings/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/general-setting/meetings/index.tsx @@ -295,28 +295,31 @@ const MeetingsSettingsMain = () => { /> - - - handleOpenPermissionSetting('screen') - } - /> - - handleOpenPermissionSetting('microphone') - } - /> - + {environment.isMacOs && ( + + + handleOpenPermissionSetting('screen') + } + /> + + handleOpenPermissionSetting('microphone') + } + /> + + )} + ({ processId, - bundleIdentifier: recording.app?.bundleIdentifier ?? 'system.global', - name: recording.app?.name ?? 'Global Recording', + bundleIdentifier: getAppBundleIdentifier(recording.app), + name: getAppName(recording.app), startTime: recording.startTime, duration: Date.now() - recording.startTime, })); @@ -294,41 +319,53 @@ function emitRecordingStatus() { io.emit('apps:recording', { recordings: getRecordingStatus() }); } -async function startRecording(app: TappableApplication) { - if (recordingMap.has(app.processId)) { +async function startRecording(app: ApplicationInfo) { + const appProcessId = getAppProcessId(app); + const appName = getAppName(app); + const appBundleId = getAppBundleIdentifier(app); + + if (recordingMap.has(appProcessId)) { console.log( - `⚠ī¸ Recording already in progress for ${app.name} (PID: ${app.processId})` + `⚠ī¸ Recording already in progress for ${appName} (PID: ${appProcessId})` ); return; } try { + console.log( + `🎙ī¸ Starting recording for ${appName} (Bundle: ${appBundleId}, PID: ${appProcessId})` + ); + const processGroupId = app.processGroupId; const rootApp = - shareableContent.applicationWithProcessId(processGroupId) || - shareableContent.applicationWithProcessId(app.processId); + ShareableContent.applicationWithProcessId(processGroupId) || + ShareableContent.applicationWithProcessId(app.processId); + if (!rootApp) { - console.error(`❌ App group not found for ${app.name}`); + console.error(`❌ App group not found for ${appName}`); return; } console.log( - `🎙ī¸ Starting recording for ${rootApp.name} (PID: ${rootApp.processId})` + `🎙ī¸ Recording from ${rootApp.name} (PID: ${rootApp.processId})` ); const buffers: Float32Array[] = []; - const session = app.tapAudio((err, samples) => { - if (err) { - console.error(`❌ Audio stream error for ${rootApp.name}:`, err); - return; + const session = ShareableContent.tapAudio( + appProcessId, + (err: any, samples: any) => { + if (err) { + console.error(`❌ Audio stream error for ${rootApp.name}:`, err); + return; + } + const recording = recordingMap.get(appProcessId); + if (recording && !recording.isWriting) { + buffers.push(new Float32Array(samples)); + } } - const recording = recordingMap.get(app.processId); - if (recording && !recording.isWriting) { - buffers.push(new Float32Array(samples)); - } - }); + ); - recordingMap.set(app.processId, { + recordingMap.set(appProcessId, { app, appGroup: rootApp, buffers, @@ -340,7 +377,7 @@ async function startRecording(app: TappableApplication) { console.log(`✅ Recording started successfully for ${rootApp.name}`); emitRecordingStatus(); } catch (error) { - console.error(`❌ Error starting recording for ${app.name}:`, error); + console.error(`❌ Error starting recording for ${appName}:`, error); } } @@ -352,9 +389,10 @@ async function stopRecording(processId: number) { } const app = recording.appGroup || recording.app; - const appName = - app?.name ?? (recording.isGlobal ? 'Global Recording' : 'Unknown App'); - const appPid = app?.processId ?? processId; + const appName = recording.isGlobal + ? 'Global Recording' + : getAppName(app) || 'Unknown App'; + const appPid = getAppProcessId(app); console.log(`⏚ī¸ Stopping recording for ${appName} (PID: ${appPid})`); console.log( @@ -529,11 +567,15 @@ async function setupRecordingsWatcher() { } } -// Application management -const shareableContent = new ShareableContent(); - +/** + * Gets all applications and groups them by bundle identifier. + * For apps with the same bundle ID (e.g., multiple processes of the same app), + * only one representative is returned. The selection prioritizes: + * 1. Running apps over stopped apps + * 2. Lower process IDs (usually parent processes) + */ async function getAllApps(): Promise { - const apps: (AppInfo | null)[] = shareableContent.applications().map(app => { + const apps: (AppInfo | null)[] = ShareableContent.applications().map(app => { try { return { app, @@ -541,7 +583,9 @@ async function getAllApps(): Promise { processGroupId: app.processGroupId, bundleIdentifier: app.bundleIdentifier, name: app.name, - isRunning: app.isRunning, + get isRunning() { + return ShareableContent.isUsingMicrophone(app.processId); + }, }; } catch (error) { console.error(error); @@ -557,37 +601,57 @@ async function getAllApps(): Promise { ); }); + // Group apps by bundleIdentifier - only keep one representative per bundle ID + const bundleGroups = new Map(); + + // Group all apps by their bundle identifier for (const app of filteredApps) { - if (filteredApps.some(a => a.processId === app.processGroupId)) { - continue; + const bundleId = app.bundleIdentifier; + if (!bundleGroups.has(bundleId)) { + bundleGroups.set(bundleId, []); } - const appGroup = shareableContent.applicationWithProcessId( - app.processGroupId - ); - if (!appGroup) { - continue; - } - filteredApps.push({ - processId: appGroup.processId, - processGroupId: appGroup.processGroupId, - bundleIdentifier: appGroup.bundleIdentifier, - name: appGroup.name, - isRunning: false, - }); + bundleGroups.get(bundleId)?.push(app); } - // Stop recording if app is not listed + console.log(`đŸ“Ļ Found ${bundleGroups.size} unique bundle identifiers`); + + // For each bundle group, select the best representative + const groupedApps: AppInfo[] = []; + + for (const [_, appsInGroup] of bundleGroups) { + if (appsInGroup.length === 1) { + // Only one app with this bundle ID, use it directly + groupedApps.push(appsInGroup[0]); + } else { + // Multiple apps with same bundle ID, choose the best representative + + // Prefer running apps, then apps with lower process IDs (usually parent processes) + const sortedApps = appsInGroup.sort((a, b) => { + // First priority: running apps + if (a.isRunning !== b.isRunning) { + return a.isRunning ? -1 : 1; + } + // Second priority: lower process ID (usually parent process) + return a.processId - b.processId; + }); + + const representative = sortedApps[0]; + groupedApps.push(representative); + } + } + + // Stop recording if app is not listed (check by process ID) await Promise.all( Array.from(recordingMap.keys()).map(async processId => { - if (!filteredApps.some(a => a.processId === processId)) { + if (!groupedApps.some(a => a.processId === processId)) { await stopRecording(processId); } }) ); - listenToAppStateChanges(filteredApps); + listenToAppStateChanges(groupedApps); - return filteredApps; + return groupedApps; } function listenToAppStateChanges(apps: AppInfo[]) { @@ -597,19 +661,27 @@ function listenToAppStateChanges(apps: AppInfo[]) { return { unsubscribe: () => {} }; } + const appName = getAppName(app); + const appProcessId = getAppProcessId(app); + const onAppStateChanged = () => { + const currentIsRunning = + ShareableContent.isUsingMicrophone(appProcessId); + console.log( - `🔄 Application state changed: ${app.name} (PID: ${app.processId}) is now ${ - app.isRunning ? 'â–ļī¸ running' : '⏚ī¸ stopped' + `🔄 Application state changed: ${appName} (PID: ${appProcessId}) is now ${ + currentIsRunning ? 'â–ļī¸ running' : '⏚ī¸ stopped' }` ); + + // Emit state change to all clients io.emit('apps:state-changed', { - processId: app.processId, - isRunning: app.isRunning, + processId: appProcessId, + isRunning: currentIsRunning, }); - if (!app.isRunning) { - stopRecording(app.processId).catch(error => { + if (!currentIsRunning) { + stopRecording(appProcessId).catch(error => { console.error('❌ Error stopping recording:', error); }); } @@ -621,7 +693,7 @@ function listenToAppStateChanges(apps: AppInfo[]) { ); } catch (error) { console.error( - `Failed to listen to app state changes for ${app?.name}:`, + `Failed to listen to app state changes for ${app ? getAppName(app) : 'unknown app'}:`, error ); return { unsubscribe: () => {} }; @@ -652,7 +724,8 @@ io.on('connection', async socket => { console.log(`📤 Sending ${files.length} saved recordings to new client`); socket.emit('apps:saved', { recordings: files }); - listenToAppStateChanges(initialApps.map(app => app.app).filter(app => !!app)); + // Set up state listeners for the current apps + listenToAppStateChanges(initialApps.filter(appInfo => appInfo.app != null)); socket.on('disconnect', () => { console.log('🔌 Client disconnected'); @@ -667,6 +740,9 @@ ShareableContent.onApplicationListChanged(() => { const apps = await getAllApps(); console.log(`đŸ“ĸ Broadcasting ${apps.length} applications to all clients`); io.emit('apps:all', { apps }); + + // Set up state listeners for the updated apps + listenToAppStateChanges(apps.filter(appInfo => appInfo.app != null)); } catch (error) { console.error('❌ Error handling application list change:', error); } @@ -769,7 +845,7 @@ app.delete('/recordings/:foldername', rateLimiter, (async ( app.get('/apps/:process_id/icon', (req, res) => { const processId = parseInt(req.params.process_id); try { - const app = shareableContent.applicationWithProcessId(processId); + const app = ShareableContent.applicationWithProcessId(processId); if (!app) { res.status(404).json({ error: 'App not found' }); return; @@ -786,7 +862,7 @@ app.get('/apps/:process_id/icon', (req, res) => { app.post('/apps/:process_id/record', async (req, res) => { const processId = parseInt(req.params.process_id); try { - const app = shareableContent.tappableApplicationWithProcessId(processId); + const app = ShareableContent.applicationWithProcessId(processId); if (!app) { res.status(404).json({ error: 'App not found' }); return; diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index 31c0b3f2d6..ae3c03e985 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -1,11 +1,12 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ -export declare class Application { - constructor(processId: number) - get processId(): number +export declare class ApplicationInfo { + processId: number + name: string + objectId: number + constructor(processId: number, name: string, objectId: number) get processGroupId(): number get bundleIdentifier(): string - get name(): string get icon(): Buffer } @@ -18,12 +19,30 @@ export declare class ApplicationStateChangedSubscriber { } export declare class AudioCaptureSession { - stop(): void get sampleRate(): number get channels(): number get actualSampleRate(): number + stop(): void } +export declare class ShareableContent { + static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber + static onAppStateChanged(app: ApplicationInfo, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber + constructor() + static applications(): Array + static applicationWithProcessId(processId: number): ApplicationInfo | null + static tapAudio(processId: number, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession + static tapGlobalAudio(excludedProcesses: Array | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession + static isUsingMicrophone(processId: number): boolean +} + +export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise + +/** Decode audio file into a Float32Array */ +export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array +export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise + +export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise export declare class DocStorage { constructor(path: string) validate(): Promise @@ -64,21 +83,43 @@ export declare class DocStoragePool { getBlobUploadedAt(universalId: string, peer: string, blobId: string): Promise } -export declare class RecordingPermissions { - audio: boolean - screen: boolean +export interface Blob { + key: string + data: Uint8Array + mime: string + size: number + createdAt: Date } -export declare class ShareableContent { - static onApplicationListChanged(callback: ((err: Error | null, ) => void)): ApplicationListChangedSubscriber - static onAppStateChanged(app: TappableApplication, callback: ((err: Error | null, ) => void)): ApplicationStateChangedSubscriber - constructor() - applications(): Array - applicationWithProcessId(processId: number): Application | null - tappableApplicationWithProcessId(processId: number): TappableApplication | null - static tapGlobalAudio(excludedProcesses: Array | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession +export interface DocClock { + docId: string + timestamp: Date } +export interface DocRecord { + docId: string + bin: Uint8Array + timestamp: Date +} + +export interface DocUpdate { + docId: string + timestamp: Date + bin: Uint8Array +} + +export interface ListedBlob { + key: string + size: number + mime: string + createdAt: Date +} + +export interface SetBlob { + key: string + data: Uint8Array + mime: string +} export declare class SqliteConnection { constructor(path: string) connect(): Promise @@ -117,80 +158,22 @@ export declare class SqliteConnection { checkpoint(): Promise } -export declare class TappableApplication { - constructor(objectId: AudioObjectID) - static fromApplication(app: Application, objectId: AudioObjectID): TappableApplication - get processId(): number - get processGroupId(): number - get bundleIdentifier(): string - get name(): string - get objectId(): number - get icon(): Buffer - get isRunning(): boolean - tapAudio(audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession -} - -export interface Blob { - key: string - data: Uint8Array - mime: string - size: number - createdAt: Date -} - export interface BlobRow { key: string data: Buffer timestamp: Date } -export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise - -/** Decode audio file into a Float32Array */ -export declare function decodeAudioSync(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null): Float32Array - -export interface DocClock { - docId: string - timestamp: Date -} - -export interface DocRecord { - docId: string - bin: Uint8Array - timestamp: Date -} - export interface DocTimestampRow { docId?: string timestamp: Date } -export interface DocUpdate { - docId: string - timestamp: Date - bin: Uint8Array -} - export interface InsertRow { docId?: string data: Uint8Array } -export interface ListedBlob { - key: string - size: number - mime: string - createdAt: Date -} - -export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise - -export interface SetBlob { - key: string - data: Uint8Array - mime: string -} - export interface UpdateRow { id: number timestamp: Date @@ -205,5 +188,3 @@ export declare enum ValidationResult { GeneralError = 3, Valid = 4 } - -export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise diff --git a/packages/frontend/native/index.js b/packages/frontend/native/index.js index c5d8f7c483..9ed7e3f8b7 100644 --- a/packages/frontend/native/index.js +++ b/packages/frontend/native/index.js @@ -365,28 +365,27 @@ if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) { if (!nativeBinding) { if (loadErrors.length > 0) { - // TODO Link to documentation with potential fixes - // - The package owner could build/publish bindings for this arch - // - The user may need to bundle the correct files - // - The user may need to re-install node_modules to get new packages - throw new Error('Failed to load native binding', { cause: loadErrors }) + throw new Error( + `Cannot find native binding. ` + + `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` + + 'Please try `npm i` again after removing both package-lock.json and node_modules directory.', + { cause: loadErrors } + ) } throw new Error(`Failed to load native binding`) } module.exports = nativeBinding -module.exports.Application = nativeBinding.Application +module.exports.ApplicationInfo = nativeBinding.ApplicationInfo module.exports.ApplicationListChangedSubscriber = nativeBinding.ApplicationListChangedSubscriber module.exports.ApplicationStateChangedSubscriber = nativeBinding.ApplicationStateChangedSubscriber module.exports.AudioCaptureSession = nativeBinding.AudioCaptureSession -module.exports.DocStorage = nativeBinding.DocStorage -module.exports.DocStoragePool = nativeBinding.DocStoragePool -module.exports.RecordingPermissions = nativeBinding.RecordingPermissions module.exports.ShareableContent = nativeBinding.ShareableContent -module.exports.SqliteConnection = nativeBinding.SqliteConnection -module.exports.TappableApplication = nativeBinding.TappableApplication module.exports.decodeAudio = nativeBinding.decodeAudio module.exports.decodeAudioSync = nativeBinding.decodeAudioSync module.exports.mintChallengeResponse = nativeBinding.mintChallengeResponse -module.exports.ValidationResult = nativeBinding.ValidationResult module.exports.verifyChallengeResponse = nativeBinding.verifyChallengeResponse +module.exports.DocStorage = nativeBinding.DocStorage +module.exports.DocStoragePool = nativeBinding.DocStoragePool +module.exports.SqliteConnection = nativeBinding.SqliteConnection +module.exports.ValidationResult = nativeBinding.ValidationResult diff --git a/packages/frontend/native/media_capture/Cargo.toml b/packages/frontend/native/media_capture/Cargo.toml index 25a798a64a..c5fffb8728 100644 --- a/packages/frontend/native/media_capture/Cargo.toml +++ b/packages/frontend/native/media_capture/Cargo.toml @@ -11,11 +11,11 @@ harness = false name = "mix_audio_samples" [dependencies] -napi = { workspace = true, features = ["napi4"] } -napi-derive = { workspace = true, features = ["type-def"] } -rubato = { workspace = true } -symphonia = { workspace = true, features = ["all", "opt-simd"] } -thiserror = { workspace = true } +napi = { workspace = true, features = ["napi4"] } +napi-derive = { workspace = true, features = ["type-def"] } +rubato = { workspace = true } +symphonia = { workspace = true, features = ["all", "opt-simd"] } +thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] block2 = { workspace = true } @@ -28,6 +28,12 @@ objc2-foundation = { workspace = true } screencapturekit = { workspace = true } uuid = { workspace = true, features = ["v4"] } +[target.'cfg(target_os = "windows")'.dependencies] +cpal = { workspace = true } +crossbeam-channel = { workspace = true } +windows = { workspace = true } +windows-core = { workspace = true } + [dev-dependencies] criterion2 = { workspace = true } diff --git a/packages/frontend/native/media_capture/src/lib.rs b/packages/frontend/native/media_capture/src/lib.rs index e83c7836fc..b65d6c7667 100644 --- a/packages/frontend/native/media_capture/src/lib.rs +++ b/packages/frontend/native/media_capture/src/lib.rs @@ -2,4 +2,10 @@ pub mod macos; #[cfg(target_os = "macos")] pub(crate) use macos::*; + +#[cfg(target_os = "windows")] +pub mod windows; +#[cfg(target_os = "windows")] +pub use windows::*; + pub mod audio_decoder; diff --git a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs index 6cf7c9711c..a41500dd62 100644 --- a/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs +++ b/packages/frontend/native/media_capture/src/macos/screen_capture_kit.rs @@ -22,7 +22,7 @@ use coreaudio::sys::{ }; use libc; use napi::{ - bindgen_prelude::{Buffer, Error, Float32Array, Result, Status}, + bindgen_prelude::{Buffer, Error, Result, Status}, threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, }; use napi_derive::napi; @@ -90,59 +90,22 @@ static NSRUNNING_APPLICATION_CLASS: LazyLock> = LazyLock::new(|| AnyClass::get(c"NSRunningApplication")); #[napi] -pub struct Application { - pub(crate) process_id: i32, - pub(crate) name: String, +#[derive(Clone)] +pub struct ApplicationInfo { + pub process_id: i32, + pub name: String, + pub object_id: u32, } #[napi] -impl Application { +impl ApplicationInfo { #[napi(constructor)] - pub fn new(process_id: i32) -> Result { - // Default values for when we can't get information - let mut app = Self { + pub fn new(process_id: i32, name: String, object_id: u32) -> Self { + Self { process_id, - name: String::new(), - }; - - // Try to populate fields using NSRunningApplication - if process_id > 0 { - // Get NSRunningApplication class - if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() { - // Get running application with PID - let running_app: *mut AnyObject = unsafe { - msg_send![ - *running_app_class, - runningApplicationWithProcessIdentifier: process_id - ] - }; - - if !running_app.is_null() { - // Get name - unsafe { - let name_ptr: *mut NSString = msg_send![running_app, localizedName]; - if !name_ptr.is_null() { - let length: usize = msg_send![name_ptr, length]; - let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String]; - - if !utf8_ptr.is_null() { - let bytes = std::slice::from_raw_parts(utf8_ptr, length); - if let Ok(s) = std::str::from_utf8(bytes) { - app.name = s.to_string(); - } - } - } - } - } - } + name, + object_id, } - - Ok(app) - } - - #[napi(getter)] - pub fn process_id(&self) -> i32 { - self.process_id } #[napi(getter)] @@ -191,12 +154,18 @@ impl Application { } } - String::new() - } + // If not available, try to get from the audio process property + if self.object_id > 0 { + if let Ok(bundle_id) = + get_process_property::(&self.object_id, kAudioProcessPropertyBundleID) + { + // Safely convert CFStringRef to Rust String + let cf_string = unsafe { CFString::wrap_under_get_rule(bundle_id) }; + return cf_string.to_string(); + } + } - #[napi(getter)] - pub fn name(&self) -> &str { - &self.name + String::new() } #[napi(getter)] @@ -341,114 +310,6 @@ impl Application { } } -#[napi] -pub struct TappableApplication { - pub(crate) app: Application, - pub(crate) object_id: AudioObjectID, -} - -#[napi] -impl TappableApplication { - #[napi(constructor)] - pub fn new(object_id: AudioObjectID) -> Result { - // Get process ID from object_id - let process_id = get_process_property(&object_id, kAudioProcessPropertyPID).unwrap_or(-1); - - // Create base Application - let app = Application::new(process_id)?; - - Ok(Self { app, object_id }) - } - - #[napi(factory)] - pub fn from_application(app: &Application, object_id: AudioObjectID) -> Self { - Self { - app: Application { - process_id: app.process_id, - name: app.name.clone(), - }, - object_id, - } - } - - #[napi(getter)] - pub fn process_id(&self) -> i32 { - self.app.process_id - } - - #[napi(getter)] - pub fn process_group_id(&self) -> i32 { - self.app.process_group_id() - } - - #[napi(getter)] - pub fn bundle_identifier(&self) -> String { - // First try to get from the Application - let app_bundle_id = self.app.bundle_identifier(); - if !app_bundle_id.is_empty() { - return app_bundle_id; - } - - // If not available, try to get from the audio process property - match get_process_property::(&self.object_id, kAudioProcessPropertyBundleID) { - Ok(bundle_id) => { - // Safely convert CFStringRef to Rust String - let cf_string = unsafe { CFString::wrap_under_create_rule(bundle_id) }; - cf_string.to_string() - } - Err(_) => { - // Return empty string if we couldn't get the bundle ID - String::new() - } - } - } - - #[napi(getter)] - pub fn name(&self) -> String { - self.app.name.clone() - } - - #[napi(getter)] - pub fn object_id(&self) -> u32 { - self.object_id - } - - #[napi(getter)] - pub fn icon(&self) -> Result { - self.app.icon() - } - - #[napi(getter)] - pub fn get_is_running(&self) -> Result { - // Use catch_unwind to prevent any panics - let result = std::panic::catch_unwind(|| { - match get_process_property(&self.object_id, kAudioProcessPropertyIsRunningInput) { - Ok(is_running) => Ok(is_running), - Err(_) => Ok(false), - } - }); - - // Handle any panics - match result { - Ok(result) => result, - Err(_) => Ok(false), - } - } - - #[napi] - pub fn tap_audio( - &self, - audio_stream_callback: Arc>, - ) -> Result { - // Use AggregateDeviceManager instead of AggregateDevice directly - // This provides automatic default device change detection - let mut device_manager = AggregateDeviceManager::new(self)?; - device_manager.start_capture(audio_stream_callback)?; - let boxed_manager = Box::new(device_manager); - Ok(AudioCaptureSession::new(boxed_manager)) - } -} - #[napi] pub struct ApplicationListChangedSubscriber { listener_block: RcBlock, @@ -538,19 +399,14 @@ pub struct ShareableContent { _inner: SCShareableContent, } -#[napi] -#[derive(Default)] -pub struct RecordingPermissions { - pub audio: bool, - pub screen: bool, -} - #[napi] impl ShareableContent { #[napi] pub fn on_application_list_changed( - callback: Arc>, + callback: ThreadsafeFunction<(), ()>, ) -> Result { + let callback_arc = Arc::new(callback); + let callback_clone = callback_arc.clone(); let callback_block: RcBlock = RcBlock::new(move |_in_number_addresses, _in_addresses: *mut c_void| { if let Err(err) = RUNNING_APPLICATIONS @@ -565,9 +421,9 @@ impl ShareableContent { *running_applications = audio_process_list(); }) { - callback.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking); + callback_clone.call(Err(err), ThreadsafeFunctionCallMode::NonBlocking); } else { - callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); + callback_clone.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); } }); @@ -598,11 +454,12 @@ impl ShareableContent { #[napi] pub fn on_app_state_changed( - app: &TappableApplication, - callback: Arc>, + app: &ApplicationInfo, + callback: ThreadsafeFunction<(), ()>, ) -> Result { let id = Uuid::new_v4(); let object_id = app.object_id; + let callback_arc = Arc::new(callback); let mut lock = APPLICATION_STATE_CHANGED_SUBSCRIBERS.write().map_err(|_| { Error::new( @@ -612,7 +469,7 @@ impl ShareableContent { })?; if let Some(subscribers) = lock.get_mut(&object_id) { - subscribers.insert(id, callback); + subscribers.insert(id, callback_arc.clone()); } else { let list_change: RcBlock = RcBlock::new(move |in_number_addresses, in_addresses: *mut c_void| { @@ -659,7 +516,7 @@ impl ShareableContent { } let subscribers = { let mut map = HashMap::new(); - map.insert(id, callback); + map.insert(id, callback_arc.clone()); map }; lock.insert(object_id, subscribers); @@ -675,7 +532,7 @@ impl ShareableContent { } #[napi] - pub fn applications(&self) -> Result> { + pub fn applications() -> Result> { let app_list = RUNNING_APPLICATIONS .read() .map_err(|_| { @@ -687,13 +544,44 @@ impl ShareableContent { .iter() .flatten() .filter_map(|id| { - let tappable_app = match TappableApplication::new(*id) { - Ok(app) => app, - Err(_) => return None, - }; + // Get process ID from object_id + let process_id = get_process_property(id, kAudioProcessPropertyPID).unwrap_or(-1); - if !tappable_app.bundle_identifier().is_empty() { - Some(tappable_app) + if process_id <= 0 { + return None; + } + + // Get application name using NSRunningApplication + let mut name = String::new(); + if let Some(running_app_class) = NSRUNNING_APPLICATION_CLASS.as_ref() { + let running_app: *mut AnyObject = unsafe { + msg_send![ + *running_app_class, + runningApplicationWithProcessIdentifier: process_id + ] + }; + + if !running_app.is_null() { + unsafe { + let name_ptr: *mut NSString = msg_send![running_app, localizedName]; + if !name_ptr.is_null() { + let length: usize = msg_send![name_ptr, length]; + let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String]; + + if !utf8_ptr.is_null() { + let bytes = std::slice::from_raw_parts(utf8_ptr, length); + if let Ok(s) = std::str::from_utf8(bytes) { + name = s.to_string(); + } + } + } + } + } + } + + let app = ApplicationInfo::new(process_id, name, *id); + if !app.bundle_identifier().is_empty() { + Some(app) } else { None } @@ -704,7 +592,13 @@ impl ShareableContent { } #[napi] - pub fn application_with_process_id(&self, process_id: u32) -> Option { + pub fn application_with_process_id(process_id: u32) -> Option { + // check if the process is tappable + let tappable = ShareableContent::tappable_application_with_process_id(process_id); + if let Some(tappable) = tappable { + return Some(tappable); + } + // Get NSRunningApplication class let running_app_class = NSRUNNING_APPLICATION_CLASS.as_ref()?; @@ -720,40 +614,107 @@ impl ShareableContent { return None; } - // Create an Application directly - Application::new(process_id as i32).ok() + // Get application name + let mut name = String::new(); + unsafe { + let name_ptr: *mut NSString = msg_send![running_app, localizedName]; + if !name_ptr.is_null() { + let length: usize = msg_send![name_ptr, length]; + let utf8_ptr: *const u8 = msg_send![name_ptr, UTF8String]; + + if !utf8_ptr.is_null() { + let bytes = std::slice::from_raw_parts(utf8_ptr, length); + if let Ok(s) = std::str::from_utf8(bytes) { + name = s.to_string(); + } + } + } + } + + // Create an ApplicationInfo with the proper name and object_id 0 (since we + // don't have audio object_id from process_id alone) + Some(ApplicationInfo::new(process_id as i32, name, 0)) } - #[napi] - pub fn tappable_application_with_process_id( - &self, - process_id: u32, - ) -> Option { + pub fn tappable_application_with_process_id(process_id: u32) -> Option { // Find the TappableApplication with this process ID in the list of running // applications - match self.applications() { + match ShareableContent::applications() { Ok(apps) => { for app in apps { - if app.process_id() == process_id as i32 { + if app.process_id == process_id as i32 { return Some(app); } } - - // If we couldn't find a TappableApplication with this process ID, create a new - // one with a default object_id of 0 (which won't be able to tap audio) - match Application::new(process_id as i32) { - Ok(app) => Some(TappableApplication::from_application(&app, 0)), - Err(_) => None, - } + None } Err(_) => None, } } + #[napi] + pub fn is_using_microphone(process_id: u32) -> Result { + if process_id == 0 { + return Ok(false); + } + + // Find the audio object ID for this process + if let Ok(app_list) = RUNNING_APPLICATIONS.read() { + if let Ok(app_list) = app_list.as_ref() { + for object_id in app_list { + let pid = get_process_property(object_id, kAudioProcessPropertyPID).unwrap_or(-1); + if pid == process_id as i32 { + // Check if the process is actively using input (microphone) + match get_process_property(object_id, kAudioProcessPropertyIsRunningInput) { + Ok(is_running) => return Ok(is_running), + Err(_) => continue, + } + } + } + } + } + + Ok(false) + } + + #[napi] + pub fn tap_audio( + process_id: u32, + audio_stream_callback: ThreadsafeFunction, + ) -> Result { + let app = ShareableContent::applications()? + .into_iter() + .find(|app| app.process_id == process_id as i32); + + if let Some(app) = app { + if app.object_id == 0 { + return Err(Error::new( + Status::GenericFailure, + "Cannot tap audio: invalid object_id", + )); + } + + // Convert ThreadsafeFunction to Arc + let callback_arc = Arc::new(audio_stream_callback); + + // Use AggregateDeviceManager instead of AggregateDevice directly + // This provides automatic default device change detection + let mut device_manager = AggregateDeviceManager::new(&app)?; + device_manager.start_capture(callback_arc)?; + let boxed_manager = Box::new(device_manager); + Ok(AudioCaptureSession::new(boxed_manager)) + } else { + Err(Error::new( + Status::GenericFailure, + "Application not found or not available for audio tapping", + )) + } + } + #[napi] pub fn tap_global_audio( - excluded_processes: Option>, - audio_stream_callback: Arc>, + excluded_processes: Option>, + audio_stream_callback: ThreadsafeFunction, ) -> Result { let excluded_object_ids = excluded_processes .unwrap_or_default() @@ -761,9 +722,12 @@ impl ShareableContent { .map(|app| app.object_id) .collect::>(); + // Convert ThreadsafeFunction to Arc + let callback_arc = Arc::new(audio_stream_callback); + // Use the new AggregateDeviceManager for automatic device adaptation let mut device_manager = AggregateDeviceManager::new_global(&excluded_object_ids)?; - device_manager.start_capture(audio_stream_callback)?; + device_manager.start_capture(callback_arc)?; let boxed_manager = Box::new(device_manager); Ok(AudioCaptureSession::new(boxed_manager)) } diff --git a/packages/frontend/native/media_capture/src/macos/tap_audio.rs b/packages/frontend/native/media_capture/src/macos/tap_audio.rs index 9b7ec92b87..b01f67e17f 100644 --- a/packages/frontend/native/media_capture/src/macos/tap_audio.rs +++ b/packages/frontend/native/media_capture/src/macos/tap_audio.rs @@ -24,9 +24,8 @@ use coreaudio::sys::{ AudioObjectRemovePropertyListenerBlock, AudioTimeStamp, OSStatus, }; use napi::{ - bindgen_prelude::Float32Array, + bindgen_prelude::{Float32Array, Result, Status}, threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, - Result, }; use napi_derive::napi; use objc2::runtime::AnyObject; @@ -38,7 +37,7 @@ use crate::{ device::get_device_uid, error::CoreAudioError, queue::create_audio_tap_queue, - screen_capture_kit::TappableApplication, + screen_capture_kit::ApplicationInfo, utils::{cfstring_from_bytes_with_nul, get_global_main_property}, }; @@ -69,7 +68,7 @@ pub struct AggregateDevice { } impl AggregateDevice { - pub fn new(app: &TappableApplication) -> Result { + pub fn new(app: &ApplicationInfo) -> Result { let object_id = app.object_id; let tap_description = CATapDescription::init_stereo_mixdown_of_processes(object_id)?; @@ -242,7 +241,7 @@ impl AggregateDevice { /// Implementation for the AggregateDevice to start processing audio pub fn start( &mut self, - audio_stream_callback: Arc>, + audio_stream_callback: Arc>, // Add original_audio_stats to ensure consistent target rate original_audio_stats: AudioStats, ) -> Result { @@ -598,13 +597,13 @@ pub struct AggregateDeviceManager { app_id: Option, excluded_processes: Vec, active_stream: Option>>>, - audio_callback: Option>>, + audio_callback: Option>>, original_audio_stats: Option, } impl AggregateDeviceManager { /// Creates a new AggregateDeviceManager for a specific application - pub fn new(app: &TappableApplication) -> Result { + pub fn new(app: &ApplicationInfo) -> Result { let device = AggregateDevice::new(app)?; Ok(Self { @@ -638,7 +637,7 @@ impl AggregateDeviceManager { /// This sets up the initial stream and listeners. pub fn start_capture( &mut self, - audio_stream_callback: Arc>, + audio_stream_callback: Arc>, ) -> Result<()> { // Store the callback for potential device switch later self.audio_callback = Some(audio_stream_callback.clone()); @@ -717,10 +716,12 @@ impl AggregateDeviceManager { }; // Create a new device with updated default devices - let result: Result = (|| { + let result: Result = { if is_app_specific { if let Some(id) = app_id { - let app = TappableApplication::new(id)?; + // For device change listener, we need to create a minimal ApplicationInfo + // We don't have the name here, so we'll use an empty string + let app = ApplicationInfo::new(id as i32, String::new(), id); AggregateDevice::new(&app) } else { Err(CoreAudioError::CreateProcessTapFailed(0).into()) @@ -728,7 +729,7 @@ impl AggregateDeviceManager { } else { AggregateDevice::create_global_tap_but_exclude_processes(&excluded_processes) } - })(); + }; // If we successfully created a new device, stop the old stream and start a new // one diff --git a/packages/frontend/native/media_capture/src/windows/README.md b/packages/frontend/native/media_capture/src/windows/README.md new file mode 100644 index 0000000000..c8cf29d924 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/README.md @@ -0,0 +1,86 @@ +# Windows Audio Recording + +This module provides Windows-specific audio recording functionality using the Windows Audio Session API (WASAPI). + +## Features + +- **Microphone Activity Detection**: Monitor when applications are using the microphone +- **Process Identification**: Identify which process is using the microphone +- **Real-time Notifications**: Get callbacks when microphone usage starts/stops + +## Usage + +### MicrophoneListener + +The `MicrophoneListener` class provides real-time monitoring of microphone usage: + +```typescript +import { MicrophoneListener } from '@affine/native'; + +const listener = new MicrophoneListener((isRunning: boolean, processName: string) => { + console.log(`Microphone ${isRunning ? 'started' : 'stopped'} by ${processName}`); +}); + +// Check current status +console.log('Is microphone currently active:', listener.is_running()); +``` + +### Callback Parameters + +The callback receives two parameters: + +- `isRunning: boolean` - Whether the microphone is currently active +- `processName: string` - Name of the process using the microphone + +## Implementation Details + +### Audio Session Monitoring + +The implementation uses Windows Audio Session API to: + +1. **Enumerate Audio Sessions**: Get all active audio sessions +2. **Monitor Session State**: Track when sessions become active/inactive +3. **Process Identification**: Map audio sessions to process names +4. **Event Handling**: Provide real-time notifications + +### COM Initialization + +The module automatically initializes COM (Component Object Model) with `COINIT_MULTITHREADED` for proper Windows API interaction. + +### Error Handling + +All Windows API errors are wrapped in `WindowsAudioError` enum and converted to NAPI errors for JavaScript consumption. + +## Cross-Platform Compatibility + +This Windows implementation maintains API compatibility with the macOS version, providing the same JavaScript interface while using Windows-specific APIs underneath. + +## Platform Requirements + +- Windows 10 or later +- Microphone access permissions +- Audio devices available + +## Dependencies + +- `windows` crate v0.61 with Audio and Process features +- `windows-core` crate v0.61 +- `napi` and `napi-derive` for JavaScript bindings + +## Technical Notes + +### Thread Safety + +The implementation uses thread-safe callbacks to JavaScript with `ThreadsafeFunction<(bool, String), ()>` to ensure proper communication between the Windows audio session monitoring thread and the JavaScript runtime. + +### Process Name Resolution + +Process names are resolved using Windows APIs: + +- `GetModuleFileNameExW` for full executable path +- `GetProcessImageFileNameW` as fallback +- Automatic extraction of filename from full path + +### Session Filtering + +The implementation automatically filters out system audio sessions (like `AudioSrv`) to focus on user applications. diff --git a/packages/frontend/native/media_capture/src/windows/audio_capture.rs b/packages/frontend/native/media_capture/src/windows/audio_capture.rs new file mode 100644 index 0000000000..a8cc87a7d4 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/audio_capture.rs @@ -0,0 +1,400 @@ +use std::{ + cell::RefCell, + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::JoinHandle, +}; + +use cpal::{ + traits::{DeviceTrait, HostTrait, StreamTrait}, + SampleRate, +}; +use crossbeam_channel::unbounded; +use napi::{ + bindgen_prelude::{Float32Array, Result}, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, + Error, Status, +}; +use napi_derive::napi; +use rubato::{FastFixedIn, PolynomialDegree, Resampler}; + +const RESAMPLER_INPUT_CHUNK: usize = 1024; // samples per channel +const TARGET_FRAME_SIZE: usize = 1024; // frame size returned to JS (in mono samples) + +struct BufferedResampler { + resampler: FastFixedIn, + channels: usize, + fifo: Vec>, // per-channel queue + initial_output_discarded: bool, // Flag to discard first output block (warm-up) +} + +impl BufferedResampler { + fn new(from_sr: f64, to_sr: f64, channels: usize) -> Self { + let ratio = to_sr / from_sr; + let resampler = FastFixedIn::::new( + ratio, + 1.0, // max_resample_ratio_relative (>= 1.0, fixed ratio) + PolynomialDegree::Linear, // balance quality/perf + RESAMPLER_INPUT_CHUNK, + channels, + ) + .expect("Failed to create FastFixedIn resampler"); + + Self { + resampler, + channels, + fifo: vec![Vec::::new(); channels], + initial_output_discarded: false, + } + } + + // Feed planar samples; returns interleaved output (may be empty) + fn feed(&mut self, planar_in: &[Vec]) -> Vec { + // Push incoming samples into fifo buffers + for (ch, data) in planar_in.iter().enumerate() { + if ch < self.fifo.len() { + self.fifo[ch].extend_from_slice(data); + } + } + + let mut interleaved_out = Vec::new(); + + while self.fifo[0].len() >= RESAMPLER_INPUT_CHUNK { + // Take exactly RESAMPLER_INPUT_CHUNK per channel + let mut chunk: Vec> = Vec::with_capacity(self.channels); + for ch in 0..self.channels { + let tail: Vec = self.fifo[ch].drain(..RESAMPLER_INPUT_CHUNK).collect(); + chunk.push(tail); + } + + if let Ok(out_blocks) = self.resampler.process(&chunk, None) { + if !out_blocks.is_empty() && out_blocks.len() == self.channels { + if !self.initial_output_discarded { + self.initial_output_discarded = true; + } else { + let out_len = out_blocks[0].len(); + for i in 0..out_len { + for ch in 0..self.channels { + interleaved_out.push(out_blocks[ch][i]); + } + } + } + } + } + } + + interleaved_out + } +} + +// Thread-local cache for resamplers keyed by (from, to, channels) +thread_local! { + static RESAMPLER_CACHE: RefCell> = RefCell::new(HashMap::new()); +} + +fn process_audio_with_resampler( + samples: Vec, + from_sample_rate: u32, + to_sample_rate: u32, +) -> Vec { + if from_sample_rate == to_sample_rate { + return samples; + } + + RESAMPLER_CACHE.with(|cache| { + let mut map = cache.borrow_mut(); + let key = (from_sample_rate, to_sample_rate, 1usize); // mono resampler + let resampler = map + .entry(key) + .or_insert_with(|| BufferedResampler::new(from_sample_rate as f64, to_sample_rate as f64, 1)); + resampler.feed(&[samples]) + }) +} + +fn to_mono(frame: &[f32]) -> f32 { + if frame.is_empty() { + return 0.0; + } + let sum: f32 = frame.iter().filter(|s| s.is_finite()).copied().sum(); + let mono = if frame.len() == 1 { + sum // already mono, no reduction needed + } else { + // For multi-channel, take the sum but don't divide by channel count + // This preserves more energy while still avoiding simple doubling + sum * 0.7 // slight reduction to prevent clipping, but preserve energy + }; + mono.clamp(-1.0, 1.0) +} + +fn mix(a: &[f32], b: &[f32]) -> Vec { + let min_len = a.len().min(b.len()); + if min_len == 0 { + return Vec::new(); + } + + const MIC_GAIN: f32 = 3.0; // Higher gain for microphone input + const LOOPBACK_GAIN: f32 = 1.5; // Moderate gain for loopback + const OVERALL_GAIN: f32 = 1.2; // Final boost + + a.iter() + .take(min_len) + .zip(b.iter().take(min_len)) + .map(|(x, y)| { + let x_clean = if x.is_finite() { *x } else { 0.0 }; + let y_clean = if y.is_finite() { *y } else { 0.0 }; + + // Apply individual gains to mic (x) and loopback (y), then mix + let mic_boosted = x_clean * MIC_GAIN; + let loopback_boosted = y_clean * LOOPBACK_GAIN; + let mixed = (mic_boosted + loopback_boosted) * OVERALL_GAIN; + + // Soft limiting using tanh for more natural sound than hard clipping + if mixed.abs() > 1.0 { + mixed.signum() * (1.0 - (-mixed.abs()).exp()) + } else { + mixed + } + }) + .collect() +} + +struct AudioBuffer { + data: Vec, +} + +#[napi] +pub struct AudioCaptureSession { + mic_stream: cpal::Stream, + lb_stream: cpal::Stream, + stopped: Arc, + sample_rate: SampleRate, + channels: u32, + jh: Option>, // background mixing thread +} + +#[napi] +impl AudioCaptureSession { + #[napi(getter)] + pub fn get_sample_rate(&self) -> f64 { + self.sample_rate.0 as f64 + } + + #[napi(getter)] + pub fn get_channels(&self) -> u32 { + self.channels + } + + #[napi(getter)] + pub fn get_actual_sample_rate(&self) -> f64 { + // For CPAL we always operate at the target rate which is sample_rate + self.sample_rate.0 as f64 + } + + #[napi] + pub fn stop(&mut self) -> Result<()> { + if self.stopped.load(Ordering::SeqCst) { + return Ok(()); + } + self + .mic_stream + .pause() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + self + .lb_stream + .pause() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + self.stopped.store(true, Ordering::SeqCst); + if let Some(jh) = self.jh.take() { + let _ = jh.join(); // ignore poison + } + Ok(()) + } +} + +impl Drop for AudioCaptureSession { + fn drop(&mut self) { + let _ = self.stop(); // Ensure cleanup even if JS forgets to call stop() + } +} + +pub fn start_recording( + audio_buffer_callback: ThreadsafeFunction, +) -> Result { + let available_hosts = cpal::available_hosts(); + let host_id = available_hosts + .first() + .ok_or_else(|| Error::new(Status::GenericFailure, "No CPAL hosts available"))?; + + let host = + cpal::host_from_id(*host_id).map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + + let mic = host + .default_input_device() + .ok_or_else(|| Error::new(Status::GenericFailure, "No default input device"))?; + let loopback_device = host + .default_output_device() + .ok_or_else(|| Error::new(Status::GenericFailure, "No default output/loopback device"))?; + + let mic_config = mic + .default_input_config() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + let lb_config = loopback_device + .default_output_config() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + + let mic_sample_rate = mic_config.sample_rate(); + let lb_sample_rate = lb_config.sample_rate(); + let target_rate = SampleRate(mic_sample_rate.min(lb_sample_rate).0); + + let mic_channels = mic_config.channels(); + let lb_channels = lb_config.channels(); + + // Convert supported configs to concrete StreamConfigs + let mic_stream_config: cpal::StreamConfig = mic_config.clone().into(); + let lb_stream_config: cpal::StreamConfig = lb_config.clone().into(); + + let stopped = Arc::new(AtomicBool::new(false)); + + // Channels for passing raw buffers between callback and mixer thread + let (tx_mic, rx_mic) = unbounded::(); + let (tx_lb, rx_lb) = unbounded::(); + + // Build microphone input stream + let mic_stream = mic + .build_input_stream( + &mic_stream_config, + move |data: &[f32], _| { + let _ = tx_mic.send(AudioBuffer { + data: data.to_vec(), + }); + }, + |err| eprintln!("CPAL mic stream error: {err}"), + None, + ) + .map_err(|e| Error::new(Status::GenericFailure, format!("build_input_stream: {e}")))?; + + // Build loopback stream by creating input stream on output device (WASAPI + // supports this) + let lb_stream = loopback_device + .build_input_stream( + &lb_stream_config, + move |data: &[f32], _| { + let _ = tx_lb.send(AudioBuffer { + data: data.to_vec(), + }); + }, + |err| eprintln!("CPAL loopback stream error: {err}"), + None, + ) + .map_err(|e| Error::new(Status::GenericFailure, format!("build_lb_stream: {e}")))?; + + let stopped_flag = stopped.clone(); + + let jh = std::thread::spawn(move || { + // Accumulators before and after resampling + let mut pre_mic: Vec = Vec::new(); + let mut pre_lb: Vec = Vec::new(); + let mut post_mic: Vec = Vec::new(); + let mut post_lb: Vec = Vec::new(); + + while !stopped_flag.load(Ordering::SeqCst) { + // Gather input from channels + while let Ok(buf) = rx_mic.try_recv() { + let mono_samples: Vec = if mic_channels == 1 { + buf.data + } else { + buf + .data + .chunks(mic_channels as usize) + .map(to_mono) + .collect() + }; + pre_mic.extend_from_slice(&mono_samples); + } + + while let Ok(buf) = rx_lb.try_recv() { + let mono_samples: Vec = if lb_channels == 1 { + buf.data + } else { + buf.data.chunks(lb_channels as usize).map(to_mono).collect() + }; + pre_lb.extend_from_slice(&mono_samples); + } + + // Resample when enough samples are available + while pre_mic.len() >= RESAMPLER_INPUT_CHUNK { + let to_resample: Vec = pre_mic.drain(..RESAMPLER_INPUT_CHUNK).collect(); + let processed = process_audio_with_resampler(to_resample, mic_sample_rate.0, target_rate.0); + if !processed.is_empty() { + post_mic.extend_from_slice(&processed); + } + } + + while pre_lb.len() >= RESAMPLER_INPUT_CHUNK { + let to_resample: Vec = pre_lb.drain(..RESAMPLER_INPUT_CHUNK).collect(); + let processed = process_audio_with_resampler(to_resample, lb_sample_rate.0, target_rate.0); + if !processed.is_empty() { + post_lb.extend_from_slice(&processed); + } + } + + // Mix when we have TARGET_FRAME_SIZE samples available from both + while post_mic.len() >= TARGET_FRAME_SIZE && post_lb.len() >= TARGET_FRAME_SIZE { + let mic_chunk: Vec = post_mic.drain(..TARGET_FRAME_SIZE).collect(); + let lb_chunk: Vec = post_lb.drain(..TARGET_FRAME_SIZE).collect(); + let mixed = mix(&mic_chunk, &lb_chunk); + if !mixed.is_empty() { + let _ = audio_buffer_callback.call( + Ok(mixed.clone().into()), + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + } + + // Prevent unbounded growth – keep some slack + const MAX_PRE: usize = RESAMPLER_INPUT_CHUNK * 10; + if pre_mic.len() > MAX_PRE { + pre_mic.drain(..pre_mic.len() - MAX_PRE); + } + if pre_lb.len() > MAX_PRE { + pre_lb.drain(..pre_lb.len() - MAX_PRE); + } + + const MAX_POST: usize = TARGET_FRAME_SIZE * 10; + if post_mic.len() > MAX_POST { + post_mic.drain(..post_mic.len() - MAX_POST); + } + if post_lb.len() > MAX_POST { + post_lb.drain(..post_lb.len() - MAX_POST); + } + + // Sleep if nothing to do + if rx_mic.is_empty() + && rx_lb.is_empty() + && post_mic.len() < TARGET_FRAME_SIZE + && post_lb.len() < TARGET_FRAME_SIZE + { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + }); + + mic_stream + .play() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + lb_stream + .play() + .map_err(|e| Error::new(Status::GenericFailure, format!("{e}")))?; + + Ok(AudioCaptureSession { + mic_stream, + lb_stream, + stopped, + sample_rate: target_rate, + channels: 1, // mono output + jh: Some(jh), + }) +} diff --git a/packages/frontend/native/media_capture/src/windows/error.rs b/packages/frontend/native/media_capture/src/windows/error.rs new file mode 100644 index 0000000000..5c6d3a8ef4 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/error.rs @@ -0,0 +1,39 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum WindowsAudioError { + #[error("Failed to initialize COM: {0}")] + ComInitializationFailed(String), + #[error("Failed to create device enumerator: {0}")] + DeviceEnumeratorCreationFailed(String), + #[error("Failed to get default audio endpoint: {0}")] + DefaultAudioEndpointFailed(String), + #[error("Failed to activate audio session manager: {0}")] + AudioSessionManagerActivationFailed(String), + #[error("Failed to register session notification: {0}")] + SessionNotificationRegistrationFailed(String), + #[error("Failed to get session enumerator: {0}")] + SessionEnumeratorFailed(String), + #[error("Failed to get session count: {0}")] + SessionCountFailed(String), + #[error("Failed to get session: {0}")] + GetSessionFailed(String), + #[error("Failed to get process ID: {0}")] + ProcessIdFailed(String), + #[error("Failed to get session state: {0}")] + SessionStateFailed(String), + #[error("Failed to register audio session notification: {0}")] + AudioSessionNotificationFailed(String), + #[error("Failed to unregister audio session notification: {0}")] + AudioSessionUnregisterFailed(String), + #[error("Failed to open process: {0}")] + ProcessOpenFailed(String), + #[error("Failed to get process name: {0}")] + ProcessNameFailed(String), +} + +impl From for napi::Error { + fn from(value: WindowsAudioError) -> Self { + napi::Error::new(napi::Status::GenericFailure, value.to_string()) + } +} diff --git a/packages/frontend/native/media_capture/src/windows/microphone_listener.rs b/packages/frontend/native/media_capture/src/windows/microphone_listener.rs new file mode 100644 index 0000000000..262b565ab9 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/microphone_listener.rs @@ -0,0 +1,645 @@ +use std::{ + ffi::OsString, + os::windows::ffi::OsStringExt, + process, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, Mutex, + }, +}; + +use napi::{ + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, + Result, +}; +use windows::{ + core::Interface, + Win32::{ + Foundation::CloseHandle, + Media::Audio::{ + eCapture, eCommunications, eConsole, AudioSessionState, AudioSessionStateActive, + IAudioSessionControl, IAudioSessionControl2, IAudioSessionEnumerator, IAudioSessionEvents, + IAudioSessionEvents_Impl, IAudioSessionManager2, IAudioSessionNotification, + IAudioSessionNotification_Impl, IMMDevice, IMMDeviceCollection, IMMDeviceEnumerator, + MMDeviceEnumerator, DEVICE_STATE_ACTIVE, + }, + System::{ + Com::{CoCreateInstance, CoInitializeEx, CLSCTX_ALL, COINIT_MULTITHREADED}, + ProcessStatus::{GetModuleFileNameExW, GetProcessImageFileNameW}, + Threading::{OpenProcess, PROCESS_QUERY_INFORMATION, PROCESS_VM_READ}, + }, + }, +}; +use windows_core::implement; + +pub struct AudioProcess { + pub process_name: String, + pub process_id: u32, + pub device_id: String, + pub device_name: String, + pub is_running: bool, +} + +pub struct AudioDevice { + pub device_id: String, + pub device_name: String, + pub is_default_communications: bool, + pub is_default_console: bool, + pub has_active_sessions: bool, +} + +// Simple struct for callback data - not a NAPI object +#[derive(Clone)] +pub struct MicrophoneActivateCallback { + pub is_running: bool, + pub process_name: String, + pub device_id: String, + pub device_name: String, +} + +#[implement(IAudioSessionEvents)] +struct SessionEvents { + process_name: String, + device_id: String, + device_name: String, + callback: Arc>, + ctrl: IAudioSessionControl, + events_ref: Arc>>, + is_running: Arc, + active_sessions: Arc, + session_is_active: AtomicBool, +} + +impl IAudioSessionEvents_Impl for SessionEvents_Impl { + fn OnChannelVolumeChanged( + &self, + _channelcount: u32, + _newchannelvolumearray: *const f32, + _changedchannel: u32, + _eventcontext: *const windows_core::GUID, + ) -> windows_core::Result<()> { + Ok(()) + } + + fn OnDisplayNameChanged( + &self, + _newdisplayname: &windows_core::PCWSTR, + _eventcontext: *const windows_core::GUID, + ) -> windows_core::Result<()> { + Ok(()) + } + + fn OnGroupingParamChanged( + &self, + _newgroupingparam: *const windows_core::GUID, + _eventcontext: *const windows_core::GUID, + ) -> windows_core::Result<()> { + Ok(()) + } + + fn OnIconPathChanged( + &self, + _newiconpath: &windows_core::PCWSTR, + _eventcontext: *const windows_core::GUID, + ) -> windows_core::Result<()> { + Ok(()) + } + + fn OnSessionDisconnected( + &self, + _disconnectreason: windows::Win32::Media::Audio::AudioSessionDisconnectReason, + ) -> windows_core::Result<()> { + if let Some(events) = self.events_ref.lock().unwrap().take() { + unsafe { self.ctrl.UnregisterAudioSessionNotification(&events)? }; + } + + // If this session was active, decrement the global counter + if self.session_is_active.swap(false, Ordering::SeqCst) { + let prev = self.active_sessions.fetch_sub(1, Ordering::SeqCst); + if prev == 1 { + // Last active session ended + self.is_running.store(false, Ordering::Relaxed); + // Notify JS side that recording has stopped + self.callback.call( + Ok(( + false, + self.process_name.clone(), + self.device_id.clone(), + self.device_name.clone(), + )), + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + } + Ok(()) + } + + fn OnSimpleVolumeChanged( + &self, + _newvolume: f32, + _newmute: windows_core::BOOL, + _eventcontext: *const windows_core::GUID, + ) -> windows_core::Result<()> { + Ok(()) + } + + fn OnStateChanged(&self, newstate: AudioSessionState) -> windows_core::Result<()> { + // Determine the new recording state for this session + let currently_recording = newstate == AudioSessionStateActive; + + // Atomically swap the flag tracking this particular session + let previously_recording = self + .session_is_active + .swap(currently_recording, Ordering::SeqCst); + + // Update the global counter accordingly + if !previously_recording && currently_recording { + // Session started + let prev = self.active_sessions.fetch_add(1, Ordering::SeqCst); + if prev == 0 { + // First active session across the whole system + self.is_running.store(true, Ordering::Relaxed); + } + } else if previously_recording && !currently_recording { + // Session stopped + let prev = self.active_sessions.fetch_sub(1, Ordering::SeqCst); + if prev == 1 { + // Last active session just stopped + self.is_running.store(false, Ordering::Relaxed); + } + } + + let overall_is_running = self.active_sessions.load(Ordering::SeqCst) > 0; + + // Notify JS side (non-blocking) + self.callback.call( + Ok(( + overall_is_running, + self.process_name.clone(), + self.device_id.clone(), + self.device_name.clone(), + )), + ThreadsafeFunctionCallMode::NonBlocking, + ); + + Ok(()) + } +} + +#[implement(IAudioSessionNotification)] +struct SessionNotifier { + _mgr: IAudioSessionManager2, // keep mgr alive + device_id: String, + device_name: String, + ctrl: Mutex>, /* keep the ctrl2 and + * events alive */ + callback: Arc>, + is_running: Arc, // Shared is_running flag + active_sessions: Arc, // Global counter of active sessions +} + +impl SessionNotifier { + fn new( + mgr: &IAudioSessionManager2, + device_id: String, + device_name: String, + callback: Arc>, + is_running: Arc, + active_sessions: Arc, + ) -> Self { + Self { + _mgr: mgr.clone(), + device_id, + device_name, + ctrl: Default::default(), + callback, + is_running, + active_sessions, + } + } + + fn refresh_state(&self, ctrl: &IAudioSessionControl) -> windows_core::Result<()> { + let ctrl2: IAudioSessionControl2 = ctrl.cast()?; + let process_id = unsafe { ctrl2.GetProcessId()? }; + + // Skip current process to avoid self-detection + if process_id == process::id() { + return Ok(()); + } + + let process_name = match get_process_name(process_id) { + Some(n) => n, + None => unsafe { ctrl2.GetDisplayName()?.to_string()? }, + }; + // Skip system-sounds session + // The `IsSystemSoundsSession` always true for unknown reason + if process_name.contains("AudioSrv") { + return Ok(()); + } + + // Active ⇒ microphone is recording + if unsafe { ctrl.GetState()? } == AudioSessionStateActive { + let mut should_notify = false; + if let Ok(mut optional_ctrl) = self.ctrl.lock() { + // Increment the active session counter. If this was the first, flip is_running + // to true. + let prev = self.active_sessions.fetch_add(1, Ordering::SeqCst); + if prev == 0 { + self.is_running.store(true, Ordering::Relaxed); + } + + let events_ref = Arc::new(Mutex::new(None)); + let events: IAudioSessionEvents = SessionEvents { + callback: self.callback.clone(), + process_name: process_name.clone(), + device_id: self.device_id.clone(), + device_name: self.device_name.clone(), + events_ref: events_ref.clone(), + ctrl: ctrl.clone(), + is_running: self.is_running.clone(), + active_sessions: self.active_sessions.clone(), + session_is_active: AtomicBool::new(true), + } + .into(); + let mut events_mut_ref = events_ref.lock().unwrap(); + *events_mut_ref = Some(events.clone()); + unsafe { ctrl.RegisterAudioSessionNotification(&events)? }; + // keep the ctrl2 alive so that the notification can be called + *optional_ctrl = Some((ctrl2, events)); + + should_notify = true; + } + + if should_notify { + self.callback.call( + Ok(( + true, + process_name, + self.device_id.clone(), + self.device_name.clone(), + )), + ThreadsafeFunctionCallMode::NonBlocking, + ); + } + return Ok(()); + } + Ok(()) + } +} + +impl IAudioSessionNotification_Impl for SessionNotifier_Impl { + fn OnSessionCreated( + &self, + ctrl_ref: windows_core::Ref<'_, windows::Win32::Media::Audio::IAudioSessionControl>, + ) -> windows_core::Result<()> { + let Some(ctrl) = ctrl_ref.as_ref() else { + return Ok(()); + }; + self.refresh_state(ctrl)?; + Ok(()) + } +} + +pub fn register_audio_device_status_callback( + is_running: Arc, + active_sessions: Arc, + callback: Arc>, +) -> windows_core::Result> { + unsafe { + let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + // Get all active capture devices + let device_collection: IMMDeviceCollection = + enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?; + + let device_count = device_collection.GetCount()?; + let mut session_notifiers = Vec::new(); + + for i in 0..device_count { + let device: IMMDevice = device_collection.Item(i)?; + + // Device identifiers + let device_id_pwstr = device.GetId()?; + let device_id = device_id_pwstr.to_string()?; + let device_name = format!("Audio Device {}", i); + + // Activate session manager for this device + let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?; + + // Create notifier for this device + let session_notifier = SessionNotifier::new( + &mgr, + device_id.clone(), + device_name.clone(), + callback.clone(), + is_running.clone(), + active_sessions.clone(), + ); + + // Enumerate existing sessions to update counters and state immediately + let list: IAudioSessionEnumerator = mgr.GetSessionEnumerator()?; + let sessions = list.GetCount()?; + for idx in 0..sessions { + let ctrl = list.GetSession(idx)?; + session_notifier.refresh_state(&ctrl)?; + } + + let session_notifier_impl: IAudioSessionNotification = session_notifier.into(); + mgr.RegisterSessionNotification(&session_notifier_impl)?; + + session_notifiers.push(session_notifier_impl); + } + + Ok(session_notifiers) + } +} + +pub struct MicrophoneListener { + _session_notifiers: Vec, // keep the session_notifiers alive + is_running: Arc, +} + +impl MicrophoneListener { + pub fn new(callback: ThreadsafeFunction<(bool, String, String, String)>) -> Self { + unsafe { + if CoInitializeEx(None, COINIT_MULTITHREADED).is_err() { + // If COM initialization fails, create a listener with empty notifiers + return Self { + is_running: Arc::new(AtomicBool::new(false)), + _session_notifiers: Vec::new(), + }; + } + }; + + let is_running = Arc::new(AtomicBool::new(false)); + let active_sessions = Arc::new(AtomicUsize::new(0)); + + let session_notifiers = match register_audio_device_status_callback( + is_running.clone(), + active_sessions.clone(), + Arc::new(callback), + ) { + Ok(notifiers) => notifiers, + Err(_) => { + // If registration fails, create a listener with empty notifiers + Vec::new() + } + }; + + Self { + is_running, + _session_notifiers: session_notifiers, + } + } + + pub fn is_running(&self) -> bool { + self.is_running.load(Ordering::Relaxed) + } + + // Static method to check if a specific process is using microphone + // This is used by TappableApplication::is_running() + pub fn is_process_using_microphone(process_id: u32) -> bool { + // Use the proven get_all_audio_processes logic + match get_all_audio_processes() { + Ok(processes) => processes + .iter() + .any(|p| p.process_id == process_id && p.is_running), + Err(_) => false, + } + } +} + +fn get_mgr_audio_session_running_status( + mgr: &IAudioSessionManager2, +) -> windows_core::Result<(bool, String)> { + let list: IAudioSessionEnumerator = unsafe { mgr.GetSessionEnumerator()? }; + let sessions = unsafe { list.GetCount()? }; + for idx in 0..sessions { + let ctrl = unsafe { list.GetSession(idx)? }; + let ctrl2: IAudioSessionControl2 = ctrl.cast()?; + let process_id = unsafe { ctrl2.GetProcessId()? }; + + // Skip current process to avoid self-detection + if process_id == process::id() { + continue; + } + + let process_name = match get_process_name(process_id) { + Some(n) => n, + None => unsafe { ctrl2.GetDisplayName()?.to_string()? }, + }; + // Skip system-sounds session + // The `IsSystemSoundsSession` always true for unknown reason + if process_name.contains("AudioSrv") { + continue; + } + + // Active ⇒ microphone is recording + if unsafe { ctrl.GetState()? } == AudioSessionStateActive { + return Ok((true, process_name)); + } + } + Ok((false, String::new())) +} + +fn get_process_name(pid: u32) -> Option { + unsafe { + // Open process with required access rights + let process_handle = + OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?; + + // Allocate a buffer large enough to hold extended-length paths (up to ~32K + // characters) instead of the legacy MAX_PATH (260) limit. + let mut buffer: Vec = std::iter::repeat(0).take(32_768).collect(); + + // Try GetModuleFileNameExW first (gives full path with extension) + let length = GetModuleFileNameExW( + Some(process_handle), + None, // NULL for the process executable + &mut buffer, + ); + + // If that fails, try GetProcessImageFileNameW + let length = if length == 0 { + GetProcessImageFileNameW(process_handle, &mut buffer) + } else { + length + }; + + // Clean up + CloseHandle(process_handle).ok()?; + + if length == 0 { + return None; + } + + // Convert to OsString then to a regular String. Truncate buffer first. + buffer.truncate(length as usize); + let os_string = OsString::from_wide(&buffer); + + // Extract the file name from the path + let path_str = os_string.to_string_lossy().to_string(); + path_str.rsplit('\\').next().map(|s| s.to_string()) + } +} + +pub fn list_audio_processes() -> Result> { + unsafe { + // Try to initialize COM, but don't fail if it's already initialized + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + }; + + let result = get_all_audio_processes() + .map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?; + + Ok(result) +} + +pub fn list_audio_devices() -> Result> { + unsafe { + // Try to initialize COM, but don't fail if it's already initialized + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + }; + + let result = get_all_audio_devices() + .map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?; + + Ok(result) +} + +fn get_all_audio_processes() -> windows_core::Result> { + unsafe { + let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + let device_collection: IMMDeviceCollection = + enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?; + + let device_count = device_collection.GetCount()?; + let mut all_processes = Vec::new(); + let current_pid = process::id(); + + for i in 0..device_count { + let device: IMMDevice = device_collection.Item(i)?; + + let device_id_pwstr = device.GetId()?; + let device_id = device_id_pwstr.to_string()?; + let device_name = format!("Audio Device {}", i); + + let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?; + let list: IAudioSessionEnumerator = mgr.GetSessionEnumerator()?; + let sessions = list.GetCount()?; + + for idx in 0..sessions { + let ctrl = list.GetSession(idx)?; + let ctrl2: IAudioSessionControl2 = ctrl.cast()?; + let process_id = ctrl2.GetProcessId()?; + + // Skip current process to avoid self-detection + if process_id == current_pid { + continue; + } + + let process_name = match get_process_name(process_id) { + Some(n) => n, + None => ctrl2.GetDisplayName()?.to_string()?, + }; + + // Skip system-sounds session + if process_name.contains("AudioSrv") { + continue; + } + + let is_running = ctrl.GetState()? == AudioSessionStateActive; + + all_processes.push(AudioProcess { + process_name, + process_id, + device_id: device_id.clone(), + device_name: device_name.clone(), + is_running, + }); + } + } + + Ok(all_processes) + } +} + +fn get_all_audio_devices() -> windows_core::Result> { + unsafe { + let enumerator: IMMDeviceEnumerator = CoCreateInstance(&MMDeviceEnumerator, None, CLSCTX_ALL)?; + + let device_collection: IMMDeviceCollection = + enumerator.EnumAudioEndpoints(eCapture, DEVICE_STATE_ACTIVE)?; + + let device_count = device_collection.GetCount()?; + let mut devices = Vec::new(); + + // Get default devices for comparison + let default_comm_device_id = enumerator + .GetDefaultAudioEndpoint(eCapture, eCommunications) + .and_then(|d| d.GetId()) + .and_then(|id| Ok(id.to_string().unwrap_or_default())) + .ok(); + let default_console_device_id = enumerator + .GetDefaultAudioEndpoint(eCapture, eConsole) + .and_then(|d| d.GetId()) + .and_then(|id| Ok(id.to_string().unwrap_or_default())) + .ok(); + + for i in 0..device_count { + let device: IMMDevice = device_collection.Item(i)?; + + let device_id_pwstr = device.GetId()?; + let device_id = device_id_pwstr.to_string()?; + let device_name = format!("Audio Device {}", i); + + let is_default_communications = default_comm_device_id.as_ref() == Some(&device_id); + let is_default_console = default_console_device_id.as_ref() == Some(&device_id); + + let mgr: IAudioSessionManager2 = device.Activate(CLSCTX_ALL, None)?; + let (has_active_sessions, _) = get_mgr_audio_session_running_status(&mgr)?; + + devices.push(AudioDevice { + device_id, + device_name, + is_default_communications, + is_default_console, + has_active_sessions, + }); + } + + Ok(devices) + } +} + +pub fn get_active_audio_processes() -> Result> { + unsafe { + // Try to initialize COM, but don't fail if it's already initialized + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + }; + + let result = get_all_audio_processes() + .map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?; + + // Filter to only return active/running processes + let active_processes = result.into_iter().filter(|p| p.is_running).collect(); + Ok(active_processes) +} + +pub fn is_process_actively_using_microphone(pid: u32) -> Result { + unsafe { + // Try to initialize COM, but don't fail if it's already initialized + let _ = CoInitializeEx(None, COINIT_MULTITHREADED); + }; + + let result = get_all_audio_processes() + .map_err(|err| napi::Error::new(napi::Status::GenericFailure, err.message()))?; + + // Check if the PID exists in the list of active processes + let is_active = result + .iter() + .any(|process| process.process_id == pid && process.is_running); + + Ok(is_active) +} diff --git a/packages/frontend/native/media_capture/src/windows/mod.rs b/packages/frontend/native/media_capture/src/windows/mod.rs new file mode 100644 index 0000000000..80a8566b50 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/mod.rs @@ -0,0 +1,17 @@ +pub mod audio_capture; +pub(crate) mod error; +pub mod microphone_listener; +pub mod screen_capture_kit; + +pub use audio_capture::*; +pub use microphone_listener::*; +pub use screen_capture_kit::*; + +#[cfg(test)] +mod tests { + #[test] + fn test_windows_module_loads() { + // Simple test to ensure the Windows module compiles and loads correctly + assert!(true); + } +} diff --git a/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs b/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs new file mode 100644 index 0000000000..0a0fe83525 --- /dev/null +++ b/packages/frontend/native/media_capture/src/windows/screen_capture_kit.rs @@ -0,0 +1,449 @@ +use std::{ + collections::HashSet, + ffi::OsString, + os::windows::ffi::OsStringExt, + sync::{ + atomic::{AtomicBool, AtomicU32, Ordering}, + Arc, LazyLock, RwLock, + }, + thread, + time::Duration, +}; + +use napi::{ + bindgen_prelude::{Buffer, Error, Result, Status}, + threadsafe_function::{ThreadsafeFunction, ThreadsafeFunctionCallMode}, +}; +use napi_derive::napi; +// Windows API imports +use windows::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; // HWND removed +use windows::Win32::System::{ + Com::{CoInitializeEx, COINIT_MULTITHREADED}, + Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, Process32FirstW, Process32NextW, PROCESSENTRY32W, TH32CS_SNAPPROCESS, + }, + ProcessStatus::{GetModuleFileNameExW, GetProcessImageFileNameW}, + Threading::{OpenProcess, PROCESS_QUERY_INFORMATION, PROCESS_VM_READ}, +}; + +// Import the function from microphone_listener +use crate::windows::microphone_listener::is_process_actively_using_microphone; + +// Type alias to match macOS API +pub type AudioObjectID = u32; + +// Global storage for running applications (Windows equivalent of macOS audio +// process list) +static RUNNING_APPLICATIONS: LazyLock>> = + LazyLock::new(|| RwLock::new(get_running_processes())); + +// Simple counter for generating unique handles +static NEXT_HANDLE: AtomicU32 = AtomicU32::new(1); + +// Global storage for active watchers +static ACTIVE_APP_WATCHERS: LazyLock< + RwLock>, Arc)>>, +> = LazyLock::new(|| RwLock::new(Vec::new())); + +static ACTIVE_LIST_WATCHERS: LazyLock< + RwLock>, Arc)>>, +> = LazyLock::new(|| RwLock::new(Vec::new())); + +// Plain struct for efficient transmission via napi-rs +#[napi] +#[derive(Clone)] +pub struct ApplicationInfo { + pub process_id: i32, + pub name: String, + pub object_id: u32, +} + +#[napi] +impl ApplicationInfo { + #[napi(constructor)] + pub fn new(process_id: i32, name: String, object_id: u32) -> Self { + Self { + process_id, + name, + object_id, + } + } + + #[napi(getter)] + pub fn process_group_id(&self) -> i32 { + // Windows doesn't have process groups like Unix, return the process ID + self.process_id + } + + #[napi(getter)] + pub fn bundle_identifier(&self) -> String { + // For Windows, return the fully-qualified path to the .exe on disk + let path = get_process_executable_path(self.process_id as u32).unwrap_or_default(); + // Escape invalid filename characters for Windows + escape_filename(&path) + } + + #[napi(getter)] + pub fn icon(&self) -> Buffer { + // For now, return empty buffer. In a full implementation, you would extract + // the icon from the executable file using Windows APIs + Buffer::from(Vec::::new()) + } +} + +#[napi] +pub struct ApplicationListChangedSubscriber { + handle: u32, + // We'll store the callback and manage it through a background thread + _callback: Arc>, +} + +#[napi] +impl ApplicationListChangedSubscriber { + #[napi] + pub fn unsubscribe(&self) -> Result<()> { + if let Ok(mut watchers) = ACTIVE_LIST_WATCHERS.write() { + if let Some(pos) = watchers + .iter() + .position(|(handle, _, _)| *handle == self.handle) + { + let (_, _, should_stop) = &watchers[pos]; + should_stop.store(true, Ordering::Relaxed); + watchers.remove(pos); + } + } + Ok(()) + } +} + +#[napi] +pub struct ApplicationStateChangedSubscriber { + handle: u32, + process_id: u32, + _callback: Arc>, +} + +#[napi] +impl ApplicationStateChangedSubscriber { + pub fn process_id(&self) -> u32 { + self.process_id + } + + #[napi] + pub fn unsubscribe(&self) { + if let Ok(mut watchers) = ACTIVE_APP_WATCHERS.write() { + if let Some(pos) = watchers + .iter() + .position(|(handle, _, _, _)| *handle == self.handle) + { + let (_, _, _, should_stop) = &watchers[pos]; + should_stop.store(true, Ordering::Relaxed); + watchers.remove(pos); + } + } + } +} + +#[napi] +pub struct ShareableContent { + // Windows doesn't need an inner SCShareableContent equivalent +} + +#[napi] +impl ShareableContent { + #[napi] + pub fn on_application_list_changed( + callback: ThreadsafeFunction<(), ()>, + ) -> Result { + let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed); + let callback_arc = Arc::new(callback); + + // Start monitoring for application list changes + start_list_monitoring(handle, callback_arc.clone()); + + Ok(ApplicationListChangedSubscriber { + handle, + _callback: callback_arc, + }) + } + + #[napi] + pub fn on_app_state_changed( + app: &ApplicationInfo, + callback: ThreadsafeFunction<(), ()>, + ) -> Result { + let handle = NEXT_HANDLE.fetch_add(1, Ordering::Relaxed); + let process_id = app.process_id as u32; + let callback_arc = Arc::new(callback); + + // Start monitoring for this specific process's microphone state + start_process_monitoring(handle, process_id, callback_arc.clone()); + + Ok(ApplicationStateChangedSubscriber { + handle, + process_id, + _callback: callback_arc, + }) + } + + #[napi(constructor)] + pub fn new() -> Self { + unsafe { + CoInitializeEx(None, COINIT_MULTITHREADED) + .ok() + .unwrap_or_else(|_| { + // COM initialization failed, but we can't return an error from + // constructor This is typically not fatal as COM might + // already be initialized + }); + } + Self {} + } + + #[napi] + pub fn applications() -> Result> { + let processes = RUNNING_APPLICATIONS.read().map_err(|_| { + Error::new( + Status::GenericFailure, + "Failed to read running applications", + ) + })?; + + let mut apps = Vec::new(); + for &process_id in processes.iter() { + let name = get_process_name(process_id).unwrap_or_else(|| format!("Process {}", process_id)); + if !name.is_empty() && name != format!("Process {}", process_id) { + let app_info = ApplicationInfo::new(process_id as i32, name, process_id); + apps.push(app_info); + } + } + Ok(apps) + } + + #[napi] + pub fn application_with_process_id(process_id: u32) -> Option { + if is_process_running(process_id) { + let name = get_process_name(process_id).unwrap_or_else(|| format!("Process {}", process_id)); + Some(ApplicationInfo::new(process_id as i32, name, process_id)) + } else { + None + } + } + + #[napi] + pub fn tap_audio( + _process_id: u32, // Currently unused - Windows captures global audio + audio_stream_callback: ThreadsafeFunction, + ) -> Result { + // On Windows with CPAL, we capture global audio (mic + loopback) + // since per-application audio tapping isn't supported the same way as macOS + crate::windows::audio_capture::start_recording(audio_stream_callback) + } + + #[napi] + pub fn tap_global_audio( + _excluded_processes: Option>, + audio_stream_callback: ThreadsafeFunction, + ) -> Result { + // Delegate to audio_capture::start_recording which handles mixing mic + + // loopback + crate::windows::audio_capture::start_recording(audio_stream_callback) + } + + #[napi] + pub fn is_using_microphone(process_id: u32) -> Result { + is_process_actively_using_microphone(process_id) + } +} + +// Re-export the concrete audio capture session implemented in audio_capture.rs +pub use crate::windows::audio_capture::AudioCaptureSession; + +// Helper function to escape invalid filename characters +fn escape_filename(path: &str) -> String { + // Replace invalid filename characters with underscores + // Invalid chars on Windows: < > : " | ? * \ spaces and control chars (0-31) + path + .chars() + .map(|c| match c { + '<' | '>' | ':' | '"' | '|' | '?' | '*' | '\\' | ' ' => '_', + c if c.is_control() => '_', + c => c, + }) + .collect::() + .to_lowercase() +} + +// Helper functions for Windows process management + +fn get_running_processes() -> Vec { + let mut processes_set = HashSet::new(); // Use HashSet to avoid duplicates from the start + unsafe { + let h_snapshot_result = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + + let h_snapshot = match h_snapshot_result { + Ok(handle) => { + if handle == INVALID_HANDLE_VALUE { + // eprintln!("CreateToolhelp32Snapshot returned INVALID_HANDLE_VALUE"); + return Vec::new(); + } + handle + } + Err(_e) => { + // eprintln!("CreateToolhelp32Snapshot failed: {:?}", e); + return Vec::new(); + } + }; + + let mut pe32 = PROCESSENTRY32W::default(); + pe32.dwSize = std::mem::size_of::() as u32; + + if Process32FirstW(h_snapshot, &mut pe32).is_ok() { + loop { + processes_set.insert(pe32.th32ProcessID); + if Process32NextW(h_snapshot, &mut pe32).is_err() { + break; + } + } + } + CloseHandle(h_snapshot).unwrap_or_else(|_e| { + // eprintln!("CloseHandle failed for snapshot: {:?}", e); + }); + } + let mut processes_vec: Vec = processes_set.into_iter().collect(); + processes_vec.sort_unstable(); // Sort for consistent ordering, though not strictly necessary for functionality + processes_vec +} + +fn is_process_running(process_id: u32) -> bool { + unsafe { + match OpenProcess(PROCESS_QUERY_INFORMATION, false, process_id) { + Ok(handle) => CloseHandle(handle).is_ok(), + Err(_) => false, + } + } +} + +fn get_process_name(pid: u32) -> Option { + unsafe { + let process_handle = + OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?; + // Allocate a buffer large enough to hold extended-length paths (up to ~32K + // characters) instead of the legacy MAX_PATH (260) limit. 32 768 is the + // maximum length supported by the Win32 APIs when the path is prefixed + // with "\\?\". + let mut buffer: Vec = std::iter::repeat(0).take(32_768).collect(); + + let length = GetModuleFileNameExW(Some(process_handle), None, &mut buffer); + CloseHandle(process_handle).ok()?; + + if length == 0 { + return None; + } + + // Truncate the buffer to the length returned by the Windows API before + // doing the UTF-16 → UTF-8 conversion. + buffer.truncate(length as usize); + let os_string = OsString::from_wide(&buffer); + let path_str = os_string.to_string_lossy().to_string(); + path_str.rsplit('\\').next().map(|s| s.to_string()) + } +} + +fn get_process_executable_path(pid: u32) -> Option { + unsafe { + let process_handle = + OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, false, pid).ok()?; + // Use a buffer that can hold extended-length paths. See rationale above. + let mut buffer: Vec = std::iter::repeat(0).take(32_768).collect(); + + let length = GetProcessImageFileNameW(process_handle, &mut buffer); + CloseHandle(process_handle).ok()?; + + if length == 0 { + return None; + } + + buffer.truncate(length as usize); + let os_string = OsString::from_wide(&buffer); + let path_str = os_string.to_string_lossy().to_string(); + Some(path_str) + } +} + +// Helper function to start monitoring a specific process +fn start_process_monitoring( + handle: u32, + process_id: u32, + callback: Arc>, +) { + let should_stop = Arc::new(AtomicBool::new(false)); + let should_stop_clone = should_stop.clone(); + + // Store the watcher info + if let Ok(mut watchers) = ACTIVE_APP_WATCHERS.write() { + watchers.push((handle, process_id, callback.clone(), should_stop.clone())); + } + + // Start monitoring thread + thread::spawn(move || { + let mut last_state = false; + + loop { + if should_stop_clone.load(Ordering::Relaxed) { + break; + } + + // Check current microphone state + let current_state = is_process_actively_using_microphone(process_id).unwrap_or(false); + + // If state changed, trigger callback + if current_state != last_state { + let _ = callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); + last_state = current_state; + } + + // Sleep for a short interval before checking again + thread::sleep(Duration::from_millis(500)); + } + }); +} + +// Helper function to start monitoring application list changes +fn start_list_monitoring(handle: u32, callback: Arc>) { + let should_stop = Arc::new(AtomicBool::new(false)); + let should_stop_clone = should_stop.clone(); + + // Store the watcher info + if let Ok(mut watchers) = ACTIVE_LIST_WATCHERS.write() { + watchers.push((handle, callback.clone(), should_stop.clone())); + } + + // Start monitoring thread + thread::spawn(move || { + let mut last_processes = get_running_processes(); + + loop { + if should_stop_clone.load(Ordering::Relaxed) { + break; + } + + // Check current process list + let current_processes = get_running_processes(); + + // If process list changed, trigger callback + if current_processes != last_processes { + let _ = callback.call(Ok(()), ThreadsafeFunctionCallMode::NonBlocking); + last_processes = current_processes; + + // Update global process list + if let Ok(mut apps) = RUNNING_APPLICATIONS.write() { + *apps = last_processes.clone(); + } + } + + // Sleep for a longer interval for process list changes + thread::sleep(Duration::from_millis(2000)); + } + }); +} diff --git a/packages/frontend/native/package.json b/packages/frontend/native/package.json index 907661787e..e0692944e2 100644 --- a/packages/frontend/native/package.json +++ b/packages/frontend/native/package.json @@ -25,7 +25,7 @@ ] }, "devDependencies": { - "@napi-rs/cli": "3.0.0-alpha.81", + "@napi-rs/cli": "3.0.0-alpha.89", "@napi-rs/whisper": "^0.0.4", "@types/node": "^22.0.0", "ava": "^6.2.0", diff --git a/yarn.lock b/yarn.lock index 3ead49eb7e..d88815f3ee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -802,7 +802,7 @@ __metadata: version: 0.0.0-use.local resolution: "@affine/native@workspace:packages/frontend/native" dependencies: - "@napi-rs/cli": "npm:3.0.0-alpha.81" + "@napi-rs/cli": "npm:3.0.0-alpha.89" "@napi-rs/whisper": "npm:^0.0.4" "@types/node": "npm:^22.0.0" ava: "npm:^6.2.0" @@ -887,7 +887,7 @@ __metadata: version: 0.0.0-use.local resolution: "@affine/server-native@workspace:packages/backend/native" dependencies: - "@napi-rs/cli": "npm:3.0.0-alpha.81" + "@napi-rs/cli": "npm:3.0.0-alpha.89" lib0: "npm:^0.2.99" tiktoken: "npm:^1.0.17" tinybench: "npm:^4.0.0" @@ -7941,9 +7941,9 @@ __metadata: languageName: node linkType: hard -"@napi-rs/cli@npm:3.0.0-alpha.81": - version: 3.0.0-alpha.81 - resolution: "@napi-rs/cli@npm:3.0.0-alpha.81" +"@napi-rs/cli@npm:3.0.0-alpha.89": + version: 3.0.0-alpha.89 + resolution: "@napi-rs/cli@npm:3.0.0-alpha.89" dependencies: "@inquirer/prompts": "npm:^7.4.0" "@napi-rs/cross-toolchain": "npm:^0.0.19" @@ -7970,7 +7970,7 @@ __metadata: bin: napi: ./dist/cli.js napi-raw: ./cli.mjs - checksum: 10/1b086706f753141d3632dd49bfeb2539c1e67af7c362da937cbd0cbad1c8578cf088d2afedc3a86302fb77e3dc7784c096081dc1b4b9e1d1a3c6bffe6308a5ff + checksum: 10/8ba4122d1bf42bf844c8304e374aa6f08a7a2804cf0d45d9a0007820076b1560cb9c8d78a91c5c3c0b8a10e474f9277fc5faab78bbe87643a2ff2027f2129b11 languageName: node linkType: hard @@ -16363,7 +16363,7 @@ __metadata: version: 0.0.0-use.local resolution: "@y-octo/node@workspace:packages/common/y-octo/node" dependencies: - "@napi-rs/cli": "npm:3.0.0-alpha.81" + "@napi-rs/cli": "npm:3.0.0-alpha.89" "@types/node": "npm:^22.14.1" "@types/prompts": "npm:^2.4.9" c8: "npm:^10.1.3"