From 1a7d8d086cd36f6ac69b3c60e8e5f7a61faa9785 Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:14:25 +0000 Subject: [PATCH 01/10] Refresh uv.lock The lock had fallen behind pyproject.toml: twine is no longer a dev dependency, and the ruff and agent-framework-devui floors moved. uv sync --frozen does not notice; uv lock --check does. Signed-off-by: danbugs --- uv.lock | 494 +------------------------------------------------------- 1 file changed, 8 insertions(+), 486 deletions(-) diff --git a/uv.lock b/uv.lock index 8be0677..3c0ecf5 100644 --- a/uv.lock +++ b/uv.lock @@ -109,15 +109,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/48/df670f75f458cc7c1752a01a394fd59c830b08172dd59cf29d73f31050f9/atheris-3.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac", size = 34858525, upload-time = "2025-11-24T23:53:59.925Z" }, ] -[[package]] -name = "backports-tarfile" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/86/72/cd9b395f25e290e633655a100af28cb253e4393396264a98bd5f5951d50f/backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991", size = 86406, upload-time = "2024-05-28T17:01:54.731Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/fa/123043af240e49752f1c4bd24da5053b6bd00cad78c2be53c0d1e8b975bc/backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34", size = 30181, upload-time = "2024-05-28T17:01:53.112Z" }, -] - [[package]] name = "build" version = "1.4.2" @@ -143,152 +134,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, ] -[[package]] -name = "cffi" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, - { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, - { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, - { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, - { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, - { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, - { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, - { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, - { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, - { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, - { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, - { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, - { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, - { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, - { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, -] - -[[package]] -name = "charset-normalizer" -version = "3.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/60/e3bec1881450851b087e301bedc3daa9377a4d45f1c26aa90b0b235e38aa/charset_normalizer-3.4.6.tar.gz", hash = "sha256:1ae6b62897110aa7c79ea2f5dd38d1abca6db663687c0b1ad9aed6f6bae3d9d6", size = 143363, upload-time = "2026-03-15T18:53:25.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e6/8c/2c56124c6dc53a774d435f985b5973bc592f42d437be58c0c92d65ae7296/charset_normalizer-3.4.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2e1d8ca8611099001949d1cdfaefc510cf0f212484fe7c565f735b68c78c3c95", size = 298751, upload-time = "2026-03-15T18:50:00.003Z" }, - { url = "https://files.pythonhosted.org/packages/86/2a/2a7db6b314b966a3bcad8c731c0719c60b931b931de7ae9f34b2839289ee/charset_normalizer-3.4.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e25369dc110d58ddf29b949377a93e0716d72a24f62bad72b2b39f155949c1fd", size = 200027, upload-time = "2026-03-15T18:50:01.702Z" }, - { url = "https://files.pythonhosted.org/packages/68/f2/0fe775c74ae25e2a3b07b01538fc162737b3e3f795bada3bc26f4d4d495c/charset_normalizer-3.4.6-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:259695e2ccc253feb2a016303543d691825e920917e31f894ca1a687982b1de4", size = 220741, upload-time = "2026-03-15T18:50:03.194Z" }, - { url = "https://files.pythonhosted.org/packages/10/98/8085596e41f00b27dd6aa1e68413d1ddda7e605f34dd546833c61fddd709/charset_normalizer-3.4.6-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dda86aba335c902b6149a02a55b38e96287157e609200811837678214ba2b1db", size = 215802, upload-time = "2026-03-15T18:50:05.859Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ce/865e4e09b041bad659d682bbd98b47fb490b8e124f9398c9448065f64fee/charset_normalizer-3.4.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51fb3c322c81d20567019778cb5a4a6f2dc1c200b886bc0d636238e364848c89", size = 207908, upload-time = "2026-03-15T18:50:07.676Z" }, - { url = "https://files.pythonhosted.org/packages/a8/54/8c757f1f7349262898c2f169e0d562b39dcb977503f18fdf0814e923db78/charset_normalizer-3.4.6-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:4482481cb0572180b6fd976a4d5c72a30263e98564da68b86ec91f0fe35e8565", size = 194357, upload-time = "2026-03-15T18:50:09.327Z" }, - { url = "https://files.pythonhosted.org/packages/6f/29/e88f2fac9218907fc7a70722b393d1bbe8334c61fe9c46640dba349b6e66/charset_normalizer-3.4.6-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39f5068d35621da2881271e5c3205125cc456f54e9030d3f723288c873a71bf9", size = 205610, upload-time = "2026-03-15T18:50:10.732Z" }, - { url = "https://files.pythonhosted.org/packages/4c/c5/21d7bb0cb415287178450171d130bed9d664211fdd59731ed2c34267b07d/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8bea55c4eef25b0b19a0337dc4e3f9a15b00d569c77211fa8cde38684f234fb7", size = 203512, upload-time = "2026-03-15T18:50:12.535Z" }, - { url = "https://files.pythonhosted.org/packages/a4/be/ce52f3c7fdb35cc987ad38a53ebcef52eec498f4fb6c66ecfe62cfe57ba2/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f0cdaecd4c953bfae0b6bb64910aaaca5a424ad9c72d85cb88417bb9814f7550", size = 195398, upload-time = "2026-03-15T18:50:14.236Z" }, - { url = "https://files.pythonhosted.org/packages/81/a0/3ab5dd39d4859a3555e5dadfc8a9fa7f8352f8c183d1a65c90264517da0e/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:150b8ce8e830eb7ccb029ec9ca36022f756986aaaa7956aad6d9ec90089338c0", size = 221772, upload-time = "2026-03-15T18:50:15.581Z" }, - { url = "https://files.pythonhosted.org/packages/04/6e/6a4e41a97ba6b2fa87f849c41e4d229449a586be85053c4d90135fe82d26/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:e68c14b04827dd76dcbd1aeea9e604e3e4b78322d8faf2f8132c7138efa340a8", size = 205759, upload-time = "2026-03-15T18:50:17.047Z" }, - { url = "https://files.pythonhosted.org/packages/db/3b/34a712a5ee64a6957bf355b01dc17b12de457638d436fdb05d01e463cd1c/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:3778fd7d7cd04ae8f54651f4a7a0bd6e39a0cf20f801720a4c21d80e9b7ad6b0", size = 216938, upload-time = "2026-03-15T18:50:18.44Z" }, - { url = "https://files.pythonhosted.org/packages/cb/05/5bd1e12da9ab18790af05c61aafd01a60f489778179b621ac2a305243c62/charset_normalizer-3.4.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:dad6e0f2e481fffdcf776d10ebee25e0ef89f16d691f1e5dee4b586375fdc64b", size = 210138, upload-time = "2026-03-15T18:50:19.852Z" }, - { url = "https://files.pythonhosted.org/packages/bd/8e/3cb9e2d998ff6b21c0a1860343cb7b83eba9cdb66b91410e18fc4969d6ab/charset_normalizer-3.4.6-cp310-cp310-win32.whl", hash = "sha256:74a2e659c7ecbc73562e2a15e05039f1e22c75b7c7618b4b574a3ea9118d1557", size = 144137, upload-time = "2026-03-15T18:50:21.505Z" }, - { url = "https://files.pythonhosted.org/packages/d8/8f/78f5489ffadb0db3eb7aff53d31c24531d33eb545f0c6f6567c25f49a5ff/charset_normalizer-3.4.6-cp310-cp310-win_amd64.whl", hash = "sha256:aa9cccf4a44b9b62d8ba8b4dd06c649ba683e4bf04eea606d2e94cfc2d6ff4d6", size = 154244, upload-time = "2026-03-15T18:50:22.81Z" }, - { url = "https://files.pythonhosted.org/packages/e4/74/e472659dffb0cadb2f411282d2d76c60da1fc94076d7fffed4ae8a93ec01/charset_normalizer-3.4.6-cp310-cp310-win_arm64.whl", hash = "sha256:e985a16ff513596f217cee86c21371b8cd011c0f6f056d0920aa2d926c544058", size = 143312, upload-time = "2026-03-15T18:50:24.074Z" }, - { url = "https://files.pythonhosted.org/packages/62/28/ff6f234e628a2de61c458be2779cb182bc03f6eec12200d4a525bbfc9741/charset_normalizer-3.4.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:82060f995ab5003a2d6e0f4ad29065b7672b6593c8c63559beefe5b443242c3e", size = 293582, upload-time = "2026-03-15T18:50:25.454Z" }, - { url = "https://files.pythonhosted.org/packages/1c/b7/b1a117e5385cbdb3205f6055403c2a2a220c5ea80b8716c324eaf75c5c95/charset_normalizer-3.4.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60c74963d8350241a79cb8feea80e54d518f72c26db618862a8f53e5023deaf9", size = 197240, upload-time = "2026-03-15T18:50:27.196Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5f/2574f0f09f3c3bc1b2f992e20bce6546cb1f17e111c5be07308dc5427956/charset_normalizer-3.4.6-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6e4333fb15c83f7d1482a76d45a0818897b3d33f00efd215528ff7c51b8e35d", size = 217363, upload-time = "2026-03-15T18:50:28.601Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d1/0ae20ad77bc949ddd39b51bf383b6ca932f2916074c95cad34ae465ab71f/charset_normalizer-3.4.6-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bc72863f4d9aba2e8fd9085e63548a324ba706d2ea2c83b260da08a59b9482de", size = 212994, upload-time = "2026-03-15T18:50:30.102Z" }, - { url = "https://files.pythonhosted.org/packages/60/ac/3233d262a310c1b12633536a07cde5ddd16985e6e7e238e9f3f9423d8eb9/charset_normalizer-3.4.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9cc4fc6c196d6a8b76629a70ddfcd4635a6898756e2d9cac5565cf0654605d73", size = 204697, upload-time = "2026-03-15T18:50:31.654Z" }, - { url = "https://files.pythonhosted.org/packages/25/3c/8a18fc411f085b82303cfb7154eed5bd49c77035eb7608d049468b53f87c/charset_normalizer-3.4.6-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:0c173ce3a681f309f31b87125fecec7a5d1347261ea11ebbb856fa6006b23c8c", size = 191673, upload-time = "2026-03-15T18:50:33.433Z" }, - { url = "https://files.pythonhosted.org/packages/ff/a7/11cfe61d6c5c5c7438d6ba40919d0306ed83c9ab957f3d4da2277ff67836/charset_normalizer-3.4.6-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c907cdc8109f6c619e6254212e794d6548373cc40e1ec75e6e3823d9135d29cc", size = 201120, upload-time = "2026-03-15T18:50:35.105Z" }, - { url = "https://files.pythonhosted.org/packages/b5/10/cf491fa1abd47c02f69687046b896c950b92b6cd7337a27e6548adbec8e4/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:404a1e552cf5b675a87f0651f8b79f5f1e6fd100ee88dc612f89aa16abd4486f", size = 200911, upload-time = "2026-03-15T18:50:36.819Z" }, - { url = "https://files.pythonhosted.org/packages/28/70/039796160b48b18ed466fde0af84c1b090c4e288fae26cd674ad04a2d703/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e3c701e954abf6fc03a49f7c579cc80c2c6cc52525340ca3186c41d3f33482ef", size = 192516, upload-time = "2026-03-15T18:50:38.228Z" }, - { url = "https://files.pythonhosted.org/packages/ff/34/c56f3223393d6ff3124b9e78f7de738047c2d6bc40a4f16ac0c9d7a1cb3c/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7a6967aaf043bceabab5412ed6bd6bd26603dae84d5cb75bf8d9a74a4959d398", size = 218795, upload-time = "2026-03-15T18:50:39.664Z" }, - { url = "https://files.pythonhosted.org/packages/e8/3b/ce2d4f86c5282191a041fdc5a4ce18f1c6bd40a5bd1f74cf8625f08d51c1/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5feb91325bbceade6afab43eb3b508c63ee53579fe896c77137ded51c6b6958e", size = 201833, upload-time = "2026-03-15T18:50:41.552Z" }, - { url = "https://files.pythonhosted.org/packages/3b/9b/b6a9f76b0fd7c5b5ec58b228ff7e85095370282150f0bd50b3126f5506d6/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:f820f24b09e3e779fe84c3c456cb4108a7aa639b0d1f02c28046e11bfcd088ed", size = 213920, upload-time = "2026-03-15T18:50:43.33Z" }, - { url = "https://files.pythonhosted.org/packages/ae/98/7bc23513a33d8172365ed30ee3a3b3fe1ece14a395e5fc94129541fc6003/charset_normalizer-3.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b35b200d6a71b9839a46b9b7fff66b6638bb52fc9658aa58796b0326595d3021", size = 206951, upload-time = "2026-03-15T18:50:44.789Z" }, - { url = "https://files.pythonhosted.org/packages/32/73/c0b86f3d1458468e11aec870e6b3feac931facbe105a894b552b0e518e79/charset_normalizer-3.4.6-cp311-cp311-win32.whl", hash = "sha256:9ca4c0b502ab399ef89248a2c84c54954f77a070f28e546a85e91da627d1301e", size = 143703, upload-time = "2026-03-15T18:50:46.103Z" }, - { url = "https://files.pythonhosted.org/packages/c6/e3/76f2facfe8eddee0bbd38d2594e709033338eae44ebf1738bcefe0a06185/charset_normalizer-3.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:a9e68c9d88823b274cf1e72f28cb5dc89c990edf430b0bfd3e2fb0785bfeabf4", size = 153857, upload-time = "2026-03-15T18:50:47.563Z" }, - { url = "https://files.pythonhosted.org/packages/e2/dc/9abe19c9b27e6cd3636036b9d1b387b78c40dedbf0b47f9366737684b4b0/charset_normalizer-3.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:97d0235baafca5f2b09cf332cc275f021e694e8362c6bb9c96fc9a0eb74fc316", size = 142751, upload-time = "2026-03-15T18:50:49.234Z" }, - { url = "https://files.pythonhosted.org/packages/e5/62/c0815c992c9545347aeea7859b50dc9044d147e2e7278329c6e02ac9a616/charset_normalizer-3.4.6-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ef7fedc7a6ecbe99969cd09632516738a97eeb8bd7258bf8a0f23114c057dab", size = 295154, upload-time = "2026-03-15T18:50:50.88Z" }, - { url = "https://files.pythonhosted.org/packages/a8/37/bdca6613c2e3c58c7421891d80cc3efa1d32e882f7c4a7ee6039c3fc951a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4ea868bc28109052790eb2b52a9ab33f3aa7adc02f96673526ff47419490e21", size = 199191, upload-time = "2026-03-15T18:50:52.658Z" }, - { url = "https://files.pythonhosted.org/packages/6c/92/9934d1bbd69f7f398b38c5dae1cbf9cc672e7c34a4adf7b17c0a9c17d15d/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:836ab36280f21fc1a03c99cd05c6b7af70d2697e374c7af0b61ed271401a72a2", size = 218674, upload-time = "2026-03-15T18:50:54.102Z" }, - { url = "https://files.pythonhosted.org/packages/af/90/25f6ab406659286be929fd89ab0e78e38aa183fc374e03aa3c12d730af8a/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f1ce721c8a7dfec21fcbdfe04e8f68174183cf4e8188e0645e92aa23985c57ff", size = 215259, upload-time = "2026-03-15T18:50:55.616Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ef/79a463eb0fff7f96afa04c1d4c51f8fc85426f918db467854bfb6a569ce3/charset_normalizer-3.4.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e28d62a8fc7a1fa411c43bd65e346f3bce9716dc51b897fbe930c5987b402d5", size = 207276, upload-time = "2026-03-15T18:50:57.054Z" }, - { url = "https://files.pythonhosted.org/packages/f7/72/d0426afec4b71dc159fa6b4e68f868cd5a3ecd918fec5813a15d292a7d10/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:530d548084c4a9f7a16ed4a294d459b4f229db50df689bfe92027452452943a0", size = 195161, upload-time = "2026-03-15T18:50:58.686Z" }, - { url = "https://files.pythonhosted.org/packages/bf/18/c82b06a68bfcb6ce55e508225d210c7e6a4ea122bfc0748892f3dc4e8e11/charset_normalizer-3.4.6-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:30f445ae60aad5e1f8bdbb3108e39f6fbc09f4ea16c815c66578878325f8f15a", size = 203452, upload-time = "2026-03-15T18:51:00.196Z" }, - { url = "https://files.pythonhosted.org/packages/44/d6/0c25979b92f8adafdbb946160348d8d44aa60ce99afdc27df524379875cb/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ac2393c73378fea4e52aa56285a3d64be50f1a12395afef9cce47772f60334c2", size = 202272, upload-time = "2026-03-15T18:51:01.703Z" }, - { url = "https://files.pythonhosted.org/packages/2e/3d/7fea3e8fe84136bebbac715dd1221cc25c173c57a699c030ab9b8900cbb7/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:90ca27cd8da8118b18a52d5f547859cc1f8354a00cd1e8e5120df3e30d6279e5", size = 195622, upload-time = "2026-03-15T18:51:03.526Z" }, - { url = "https://files.pythonhosted.org/packages/57/8a/d6f7fd5cb96c58ef2f681424fbca01264461336d2a7fc875e4446b1f1346/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e5a94886bedca0f9b78fecd6afb6629142fd2605aa70a125d49f4edc6037ee6", size = 220056, upload-time = "2026-03-15T18:51:05.269Z" }, - { url = "https://files.pythonhosted.org/packages/16/50/478cdda782c8c9c3fb5da3cc72dd7f331f031e7f1363a893cdd6ca0f8de0/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:695f5c2823691a25f17bc5d5ffe79fa90972cc34b002ac6c843bb8a1720e950d", size = 203751, upload-time = "2026-03-15T18:51:06.858Z" }, - { url = "https://files.pythonhosted.org/packages/75/fc/cc2fcac943939c8e4d8791abfa139f685e5150cae9f94b60f12520feaa9b/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:231d4da14bcd9301310faf492051bee27df11f2bc7549bc0bb41fef11b82daa2", size = 216563, upload-time = "2026-03-15T18:51:08.564Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b7/a4add1d9a5f68f3d037261aecca83abdb0ab15960a3591d340e829b37298/charset_normalizer-3.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a056d1ad2633548ca18ffa2f85c202cfb48b68615129143915b8dc72a806a923", size = 209265, upload-time = "2026-03-15T18:51:10.312Z" }, - { url = "https://files.pythonhosted.org/packages/6c/18/c094561b5d64a24277707698e54b7f67bd17a4f857bbfbb1072bba07c8bf/charset_normalizer-3.4.6-cp312-cp312-win32.whl", hash = "sha256:c2274ca724536f173122f36c98ce188fd24ce3dad886ec2b7af859518ce008a4", size = 144229, upload-time = "2026-03-15T18:51:11.694Z" }, - { url = "https://files.pythonhosted.org/packages/ab/20/0567efb3a8fd481b8f34f739ebddc098ed062a59fed41a8d193a61939e8f/charset_normalizer-3.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:c8ae56368f8cc97c7e40a7ee18e1cedaf8e780cd8bc5ed5ac8b81f238614facb", size = 154277, upload-time = "2026-03-15T18:51:13.004Z" }, - { url = "https://files.pythonhosted.org/packages/15/57/28d79b44b51933119e21f65479d0864a8d5893e494cf5daab15df0247c17/charset_normalizer-3.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:899d28f422116b08be5118ef350c292b36fc15ec2daeb9ea987c89281c7bb5c4", size = 142817, upload-time = "2026-03-15T18:51:14.408Z" }, - { url = "https://files.pythonhosted.org/packages/1e/1d/4fdabeef4e231153b6ed7567602f3b68265ec4e5b76d6024cf647d43d981/charset_normalizer-3.4.6-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:11afb56037cbc4b1555a34dd69151e8e069bee82e613a73bef6e714ce733585f", size = 294823, upload-time = "2026-03-15T18:51:15.755Z" }, - { url = "https://files.pythonhosted.org/packages/47/7b/20e809b89c69d37be748d98e84dce6820bf663cf19cf6b942c951a3e8f41/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:423fb7e748a08f854a08a222b983f4df1912b1daedce51a72bd24fe8f26a1843", size = 198527, upload-time = "2026-03-15T18:51:17.177Z" }, - { url = "https://files.pythonhosted.org/packages/37/a6/4f8d27527d59c039dce6f7622593cdcd3d70a8504d87d09eb11e9fdc6062/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d73beaac5e90173ac3deb9928a74763a6d230f494e4bfb422c217a0ad8e629bf", size = 218388, upload-time = "2026-03-15T18:51:18.934Z" }, - { url = "https://files.pythonhosted.org/packages/f6/9b/4770ccb3e491a9bacf1c46cc8b812214fe367c86a96353ccc6daf87b01ec/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d60377dce4511655582e300dc1e5a5f24ba0cb229005a1d5c8d0cb72bb758ab8", size = 214563, upload-time = "2026-03-15T18:51:20.374Z" }, - { url = "https://files.pythonhosted.org/packages/2b/58/a199d245894b12db0b957d627516c78e055adc3a0d978bc7f65ddaf7c399/charset_normalizer-3.4.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:530e8cebeea0d76bdcf93357aa5e41336f48c3dc709ac52da2bb167c5b8271d9", size = 206587, upload-time = "2026-03-15T18:51:21.807Z" }, - { url = "https://files.pythonhosted.org/packages/7e/70/3def227f1ec56f5c69dfc8392b8bd63b11a18ca8178d9211d7cc5e5e4f27/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:a26611d9987b230566f24a0a125f17fe0de6a6aff9f25c9f564aaa2721a5fb88", size = 194724, upload-time = "2026-03-15T18:51:23.508Z" }, - { url = "https://files.pythonhosted.org/packages/58/ab/9318352e220c05efd31c2779a23b50969dc94b985a2efa643ed9077bfca5/charset_normalizer-3.4.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:34315ff4fc374b285ad7f4a0bf7dcbfe769e1b104230d40f49f700d4ab6bbd84", size = 202956, upload-time = "2026-03-15T18:51:25.239Z" }, - { url = "https://files.pythonhosted.org/packages/75/13/f3550a3ac25b70f87ac98c40d3199a8503676c2f1620efbf8d42095cfc40/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5f8ddd609f9e1af8c7bd6e2aca279c931aefecd148a14402d4e368f3171769fd", size = 201923, upload-time = "2026-03-15T18:51:26.682Z" }, - { url = "https://files.pythonhosted.org/packages/1b/db/c5c643b912740b45e8eec21de1bbab8e7fc085944d37e1e709d3dcd9d72f/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:80d0a5615143c0b3225e5e3ef22c8d5d51f3f72ce0ea6fb84c943546c7b25b6c", size = 195366, upload-time = "2026-03-15T18:51:28.129Z" }, - { url = "https://files.pythonhosted.org/packages/5a/67/3b1c62744f9b2448443e0eb160d8b001c849ec3fef591e012eda6484787c/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:92734d4d8d187a354a556626c221cd1a892a4e0802ccb2af432a1d85ec012194", size = 219752, upload-time = "2026-03-15T18:51:29.556Z" }, - { url = "https://files.pythonhosted.org/packages/f6/98/32ffbaf7f0366ffb0445930b87d103f6b406bc2c271563644bde8a2b1093/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:613f19aa6e082cf96e17e3ffd89383343d0d589abda756b7764cf78361fd41dc", size = 203296, upload-time = "2026-03-15T18:51:30.921Z" }, - { url = "https://files.pythonhosted.org/packages/41/12/5d308c1bbe60cabb0c5ef511574a647067e2a1f631bc8634fcafaccd8293/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:2b1a63e8224e401cafe7739f77efd3f9e7f5f2026bda4aead8e59afab537784f", size = 215956, upload-time = "2026-03-15T18:51:32.399Z" }, - { url = "https://files.pythonhosted.org/packages/53/e9/5f85f6c5e20669dbe56b165c67b0260547dea97dba7e187938833d791687/charset_normalizer-3.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cceb5473417d28edd20c6c984ab6fee6c6267d38d906823ebfe20b03d607dc2", size = 208652, upload-time = "2026-03-15T18:51:34.214Z" }, - { url = "https://files.pythonhosted.org/packages/f1/11/897052ea6af56df3eef3ca94edafee410ca699ca0c7b87960ad19932c55e/charset_normalizer-3.4.6-cp313-cp313-win32.whl", hash = "sha256:d7de2637729c67d67cf87614b566626057e95c303bc0a55ffe391f5205e7003d", size = 143940, upload-time = "2026-03-15T18:51:36.15Z" }, - { url = "https://files.pythonhosted.org/packages/a1/5c/724b6b363603e419829f561c854b87ed7c7e31231a7908708ac086cdf3e2/charset_normalizer-3.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:572d7c822caf521f0525ba1bce1a622a0b85cf47ffbdae6c9c19e3b5ac3c4389", size = 154101, upload-time = "2026-03-15T18:51:37.876Z" }, - { url = "https://files.pythonhosted.org/packages/01/a5/7abf15b4c0968e47020f9ca0935fb3274deb87cb288cd187cad92e8cdffd/charset_normalizer-3.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a4474d924a47185a06411e0064b803c68be044be2d60e50e8bddcc2649957c1f", size = 143109, upload-time = "2026-03-15T18:51:39.565Z" }, - { url = "https://files.pythonhosted.org/packages/25/6f/ffe1e1259f384594063ea1869bfb6be5cdb8bc81020fc36c3636bc8302a1/charset_normalizer-3.4.6-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:9cc6e6d9e571d2f863fa77700701dae73ed5f78881efc8b3f9a4398772ff53e8", size = 294458, upload-time = "2026-03-15T18:51:41.134Z" }, - { url = "https://files.pythonhosted.org/packages/56/60/09bb6c13a8c1016c2ed5c6a6488e4ffef506461aa5161662bd7636936fb1/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5960d965e67165d75b7c7ffc60a83ec5abfc5c11b764ec13ea54fbef8b4421", size = 199277, upload-time = "2026-03-15T18:51:42.953Z" }, - { url = "https://files.pythonhosted.org/packages/00/50/dcfbb72a5138bbefdc3332e8d81a23494bf67998b4b100703fd15fa52d81/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b3694e3f87f8ac7ce279d4355645b3c878d24d1424581b46282f24b92f5a4ae2", size = 218758, upload-time = "2026-03-15T18:51:44.339Z" }, - { url = "https://files.pythonhosted.org/packages/03/b3/d79a9a191bb75f5aa81f3aaaa387ef29ce7cb7a9e5074ba8ea095cc073c2/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d11595abf8dd942a77883a39d81433739b287b6aa71620f15164f8096221b30", size = 215299, upload-time = "2026-03-15T18:51:45.871Z" }, - { url = "https://files.pythonhosted.org/packages/76/7e/bc8911719f7084f72fd545f647601ea3532363927f807d296a8c88a62c0d/charset_normalizer-3.4.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7bda6eebafd42133efdca535b04ccb338ab29467b3f7bf79569883676fc628db", size = 206811, upload-time = "2026-03-15T18:51:47.308Z" }, - { url = "https://files.pythonhosted.org/packages/e2/40/c430b969d41dda0c465aa36cc7c2c068afb67177bef50905ac371b28ccc7/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:bbc8c8650c6e51041ad1be191742b8b421d05bbd3410f43fa2a00c8db87678e8", size = 193706, upload-time = "2026-03-15T18:51:48.849Z" }, - { url = "https://files.pythonhosted.org/packages/48/15/e35e0590af254f7df984de1323640ef375df5761f615b6225ba8deb9799a/charset_normalizer-3.4.6-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22c6f0c2fbc31e76c3b8a86fba1a56eda6166e238c29cdd3d14befdb4a4e4815", size = 202706, upload-time = "2026-03-15T18:51:50.257Z" }, - { url = "https://files.pythonhosted.org/packages/5e/bd/f736f7b9cc5e93a18b794a50346bb16fbfd6b37f99e8f306f7951d27c17c/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7edbed096e4a4798710ed6bc75dcaa2a21b68b6c356553ac4823c3658d53743a", size = 202497, upload-time = "2026-03-15T18:51:52.012Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ba/2cc9e3e7dfdf7760a6ed8da7446d22536f3d0ce114ac63dee2a5a3599e62/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7f9019c9cb613f084481bd6a100b12e1547cf2efe362d873c2e31e4035a6fa43", size = 193511, upload-time = "2026-03-15T18:51:53.723Z" }, - { url = "https://files.pythonhosted.org/packages/9e/cb/5be49b5f776e5613be07298c80e1b02a2d900f7a7de807230595c85a8b2e/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:58c948d0d086229efc484fe2f30c2d382c86720f55cd9bc33591774348ad44e0", size = 220133, upload-time = "2026-03-15T18:51:55.333Z" }, - { url = "https://files.pythonhosted.org/packages/83/43/99f1b5dad345accb322c80c7821071554f791a95ee50c1c90041c157ae99/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:419a9d91bd238052642a51938af8ac05da5b3343becde08d5cdeab9046df9ee1", size = 203035, upload-time = "2026-03-15T18:51:56.736Z" }, - { url = "https://files.pythonhosted.org/packages/87/9a/62c2cb6a531483b55dddff1a68b3d891a8b498f3ca555fbcf2978e804d9d/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:5273b9f0b5835ff0350c0828faea623c68bfa65b792720c453e22b25cc72930f", size = 216321, upload-time = "2026-03-15T18:51:58.17Z" }, - { url = "https://files.pythonhosted.org/packages/6e/79/94a010ff81e3aec7c293eb82c28f930918e517bc144c9906a060844462eb/charset_normalizer-3.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0e901eb1049fdb80f5bd11ed5ea1e498ec423102f7a9b9e4645d5b8204ff2815", size = 208973, upload-time = "2026-03-15T18:51:59.998Z" }, - { url = "https://files.pythonhosted.org/packages/2a/57/4ecff6d4ec8585342f0c71bc03efaa99cb7468f7c91a57b105bcd561cea8/charset_normalizer-3.4.6-cp314-cp314-win32.whl", hash = "sha256:b4ff1d35e8c5bd078be89349b6f3a845128e685e751b6ea1169cf2160b344c4d", size = 144610, upload-time = "2026-03-15T18:52:02.213Z" }, - { url = "https://files.pythonhosted.org/packages/80/94/8434a02d9d7f168c25767c64671fead8d599744a05d6a6c877144c754246/charset_normalizer-3.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:74119174722c4349af9708993118581686f343adc1c8c9c007d59be90d077f3f", size = 154962, upload-time = "2026-03-15T18:52:03.658Z" }, - { url = "https://files.pythonhosted.org/packages/46/4c/48f2cdbfd923026503dfd67ccea45c94fd8fe988d9056b468579c66ed62b/charset_normalizer-3.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:e5bcc1a1ae744e0bb59641171ae53743760130600da8db48cbb6e4918e186e4e", size = 143595, upload-time = "2026-03-15T18:52:05.123Z" }, - { url = "https://files.pythonhosted.org/packages/31/93/8878be7569f87b14f1d52032946131bcb6ebbd8af3e20446bc04053dc3f1/charset_normalizer-3.4.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:ad8faf8df23f0378c6d527d8b0b15ea4a2e23c89376877c598c4870d1b2c7866", size = 314828, upload-time = "2026-03-15T18:52:06.831Z" }, - { url = "https://files.pythonhosted.org/packages/06/b6/fae511ca98aac69ecc35cde828b0a3d146325dd03d99655ad38fc2cc3293/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f5ea69428fa1b49573eef0cc44a1d43bebd45ad0c611eb7d7eac760c7ae771bc", size = 208138, upload-time = "2026-03-15T18:52:08.239Z" }, - { url = "https://files.pythonhosted.org/packages/54/57/64caf6e1bf07274a1e0b7c160a55ee9e8c9ec32c46846ce59b9c333f7008/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:06a7e86163334edfc5d20fe104db92fcd666e5a5df0977cb5680a506fe26cc8e", size = 224679, upload-time = "2026-03-15T18:52:10.043Z" }, - { url = "https://files.pythonhosted.org/packages/aa/cb/9ff5a25b9273ef160861b41f6937f86fae18b0792fe0a8e75e06acb08f1d/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e1f6e2f00a6b8edb562826e4632e26d063ac10307e80f7461f7de3ad8ef3f077", size = 223475, upload-time = "2026-03-15T18:52:11.854Z" }, - { url = "https://files.pythonhosted.org/packages/fc/97/440635fc093b8d7347502a377031f9605a1039c958f3cd18dcacffb37743/charset_normalizer-3.4.6-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95b52c68d64c1878818687a473a10547b3292e82b6f6fe483808fb1468e2f52f", size = 215230, upload-time = "2026-03-15T18:52:13.325Z" }, - { url = "https://files.pythonhosted.org/packages/cd/24/afff630feb571a13f07c8539fbb502d2ab494019492aaffc78ef41f1d1d0/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:7504e9b7dc05f99a9bbb4525c67a2c155073b44d720470a148b34166a69c054e", size = 199045, upload-time = "2026-03-15T18:52:14.752Z" }, - { url = "https://files.pythonhosted.org/packages/e5/17/d1399ecdaf7e0498c327433e7eefdd862b41236a7e484355b8e0e5ebd64b/charset_normalizer-3.4.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:172985e4ff804a7ad08eebec0a1640ece87ba5041d565fff23c8f99c1f389484", size = 211658, upload-time = "2026-03-15T18:52:16.278Z" }, - { url = "https://files.pythonhosted.org/packages/b5/38/16baa0affb957b3d880e5ac2144caf3f9d7de7bc4a91842e447fbb5e8b67/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4be9f4830ba8741527693848403e2c457c16e499100963ec711b1c6f2049b7c7", size = 210769, upload-time = "2026-03-15T18:52:17.782Z" }, - { url = "https://files.pythonhosted.org/packages/05/34/c531bc6ac4c21da9ddfddb3107be2287188b3ea4b53b70fc58f2a77ac8d8/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:79090741d842f564b1b2827c0b82d846405b744d31e84f18d7a7b41c20e473ff", size = 201328, upload-time = "2026-03-15T18:52:19.553Z" }, - { url = "https://files.pythonhosted.org/packages/fa/73/a5a1e9ca5f234519c1953608a03fe109c306b97fdfb25f09182babad51a7/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87725cfb1a4f1f8c2fc9890ae2f42094120f4b44db9360be5d99a4c6b0e03a9e", size = 225302, upload-time = "2026-03-15T18:52:21.043Z" }, - { url = "https://files.pythonhosted.org/packages/ba/f6/cd782923d112d296294dea4bcc7af5a7ae0f86ab79f8fefbda5526b6cfc0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fcce033e4021347d80ed9c66dcf1e7b1546319834b74445f561d2e2221de5659", size = 211127, upload-time = "2026-03-15T18:52:22.491Z" }, - { url = "https://files.pythonhosted.org/packages/0e/c5/0b6898950627af7d6103a449b22320372c24c6feda91aa24e201a478d161/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ca0276464d148c72defa8bb4390cce01b4a0e425f3b50d1435aa6d7a18107602", size = 222840, upload-time = "2026-03-15T18:52:24.113Z" }, - { url = "https://files.pythonhosted.org/packages/7d/25/c4bba773bef442cbdc06111d40daa3de5050a676fa26e85090fc54dd12f0/charset_normalizer-3.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:197c1a244a274bb016dd8b79204850144ef77fe81c5b797dc389327adb552407", size = 216890, upload-time = "2026-03-15T18:52:25.541Z" }, - { url = "https://files.pythonhosted.org/packages/35/1a/05dacadb0978da72ee287b0143097db12f2e7e8d3ffc4647da07a383b0b7/charset_normalizer-3.4.6-cp314-cp314t-win32.whl", hash = "sha256:2a24157fa36980478dd1770b585c0f30d19e18f4fb0c47c13aa568f871718579", size = 155379, upload-time = "2026-03-15T18:52:27.05Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7a/d269d834cb3a76291651256f3b9a5945e81d0a49ab9f4a498964e83c0416/charset_normalizer-3.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:cd5e2801c89992ed8c0a3f0293ae83c159a60d9a5d685005383ef4caca77f2c4", size = 169043, upload-time = "2026-03-15T18:52:28.502Z" }, - { url = "https://files.pythonhosted.org/packages/23/06/28b29fba521a37a8932c6a84192175c34d49f84a6d4773fa63d05f9aff22/charset_normalizer-3.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:47955475ac79cc504ef2704b192364e51d0d473ad452caedd0002605f780101c", size = 148523, upload-time = "2026-03-15T18:52:29.956Z" }, - { url = "https://files.pythonhosted.org/packages/2a/68/687187c7e26cb24ccbd88e5069f5ef00eba804d36dde11d99aad0838ab45/charset_normalizer-3.4.6-py3-none-any.whl", hash = "sha256:947cf925bc916d90adba35a64c82aace04fa39b46b52d4630ece166655905a69", size = 61455, upload-time = "2026-03-15T18:53:23.833Z" }, -] - [[package]] name = "click" version = "8.3.1" @@ -323,49 +168,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/bd/ee1ffcc1d656cc93eabbd826bccba3b235068fb9cba89749a24b4a32f6f6/componentize_py-0.21.0-cp39-abi3-win_amd64.whl", hash = "sha256:ef38862a8bdb3574b46848f8f8cb7c6c4cd85067f844fad7b24c08ed022e227c", size = 22154053, upload-time = "2026-02-12T23:17:46.149Z" }, ] -[[package]] -name = "cryptography" -version = "50.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, - { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, - { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, - { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, - { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, - { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, - { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, - { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, - { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, - { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, - { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, - { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, - { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, - { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, - { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, - { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, - { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, - { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, - { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, - { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, - { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, - { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, - { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, - { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, - { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, - { url = "https://files.pythonhosted.org/packages/01/b6/0b9e125e90f3d2dcf599a218a899cda7326a3158cfa258723f0b398b08f6/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a", size = 4692441, upload-time = "2026-07-31T14:24:53.743Z" }, - { url = "https://files.pythonhosted.org/packages/53/c9/a5151588710785a96d7bc4de27d4cd62f263bbbcb203cfe29df537eb6505/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e", size = 4699810, upload-time = "2026-07-31T14:24:55.746Z" }, - { url = "https://files.pythonhosted.org/packages/c7/1a/15b92b25eb6ce3089cd49377ae990a0f3ad485a510f968aed1f19dbdcdf2/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d", size = 4691924, upload-time = "2026-07-31T14:24:58.082Z" }, - { url = "https://files.pythonhosted.org/packages/62/15/219075012ab13e8905f3cd572204f4acb4b111df787104346b9bc0cea789/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437", size = 4699593, upload-time = "2026-07-31T14:24:59.951Z" }, -] - [[package]] name = "distro" version = "1.9.0" @@ -375,21 +177,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "docutils" -version = "0.22.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, -] - [[package]] name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -420,9 +213,9 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "pydantic" }, - { name = "python-dateutil" }, - { name = "typing-extensions" }, + { name = "pydantic", marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/87/06/1dec504b54c724d69283969d4ed004225ec8bbb1c0a5e9e0c3b6b048099a/github_copilot_sdk-0.1.25-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:d32c3fc2c393f70923a645a133607da2e562d078b87437f499100d5bb8c1902f", size = 58097936, upload-time = "2026-02-18T00:07:20.672Z" }, @@ -441,8 +234,8 @@ resolution-markers = [ "python_full_version >= '3.11'", ] dependencies = [ - { name = "pydantic" }, - { name = "python-dateutil" }, + { name = "pydantic", marker = "python_full_version >= '3.11'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1f/2c/3d3ecfe500c0ba7d3127737b1aa22f19ff1a19e6e86360bfdca3f02a2c09/github_copilot_sdk-1.0.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:856dfc8370f36f6efd8a2aa1dd40f82c1a6d0573d0577eaff1f6affb73ed29ad", size = 97329153, upload-time = "2026-06-18T00:56:20.653Z" }, @@ -619,7 +412,7 @@ agent-framework = [ { name = "hyperlight-sandbox-python-guest", editable = "src/sdk/python/wasm_guests/python_guest" }, { name = "pydantic" }, ] -agent-framework-devui = [{ name = "agent-framework-devui", specifier = ">=1.0.0b260903" }] +agent-framework-devui = [{ name = "agent-framework-devui", specifier = ">=1.0.0b261002" }] copilot-sdk = [ { name = "github-copilot-sdk" }, { name = "hyperlight-sandbox", editable = "src/sdk/python/core" }, @@ -634,7 +427,7 @@ dev = [ { name = "hyperlight-sandbox-javascript-guest", editable = "src/sdk/python/wasm_guests/javascript_guest" }, { name = "hyperlight-sandbox-python-guest", editable = "src/sdk/python/wasm_guests/python_guest" }, { name = "maturin", specifier = ">=1.15.0,<2.0" }, - { name = "ruff", specifier = ">=0.16.6" }, + { name = "ruff", specifier = ">=0.16.10" }, ] [[package]] @@ -647,18 +440,6 @@ name = "hyperlight-sandbox-python-guest" version = "0.8.0" source = { editable = "src/sdk/python/wasm_guests/python_guest" } -[[package]] -name = "id" -version = "1.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/04/c2156091427636080787aac190019dc64096e56a23b7364d3c1764ee3a06/id-1.6.1.tar.gz", hash = "sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069", size = 18088, upload-time = "2026-02-04T16:19:41.26Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/77/de194443bf38daed9452139e960c632b0ef9f9a5dd9ce605fdf18ca9f1b1/id-1.6.1-py3-none-any.whl", hash = "sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca", size = 14689, upload-time = "2026-02-04T16:19:40.051Z" }, -] - [[package]] name = "idna" version = "3.17" @@ -680,51 +461,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/5e/f8e9a1d23b9c20a551a8a02ea3637b4642e22c2626e3a13a9a29cdea99eb/importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151", size = 27865, upload-time = "2025-12-21T10:00:18.329Z" }, ] -[[package]] -name = "jaraco-classes" -version = "3.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, -] - -[[package]] -name = "jaraco-context" -version = "6.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, -] - -[[package]] -name = "jaraco-functools" -version = "4.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "more-itertools" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0f/27/056e0638a86749374d6f57d0b0db39f29509cce9313cf91bdc0ac4d91084/jaraco_functools-4.4.0.tar.gz", hash = "sha256:da21933b0417b89515562656547a77b4931f98176eb173644c0d35032a33d6bb", size = 19943, upload-time = "2025-12-21T09:29:43.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/c4/813bb09f0985cb21e959f21f2464169eca882656849adf727ac7bb7e1767/jaraco_functools-4.4.0-py3-none-any.whl", hash = "sha256:9eec1e36f45c818d9bf307c8948eb03b2b56cd44087b3cdc989abca1f20b9176", size = 10481, upload-time = "2025-12-21T09:29:42.27Z" }, -] - -[[package]] -name = "jeepney" -version = "0.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, -] - [[package]] name = "jiter" version = "0.13.0" @@ -822,36 +558,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/8a/a342b2f0251f3dac4ca17618265d93bf244a2a4d089126e81e4c1056ac50/jiter-0.13.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bb00b6d26db67a05fe3e12c76edc75f32077fb51deed13822dc648fa373bc19", size = 343768, upload-time = "2026-02-02T12:37:55.055Z" }, ] -[[package]] -name = "keyring" -version = "25.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, - { name = "jaraco-classes" }, - { name = "jaraco-context" }, - { name = "jaraco-functools" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, -] - [[package]] name = "maturin" version = "1.15.0" @@ -876,24 +582,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, ] -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - -[[package]] -name = "more-itertools" -version = "10.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, -] - [[package]] name = "msgspec" version = "0.21.1" @@ -950,40 +638,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/90/91/56c5d560f20e6c20e9e4f55bd0e458f7f162aa689ee350346c04c48eac0b/msgspec-0.21.1-cp314-cp314t-win_arm64.whl", hash = "sha256:0d2cc73df6058d811a126ac3a8ad63a4dfa210c82f9cf5a004802eaf4712de90", size = 183149, upload-time = "2026-04-12T21:44:48.833Z" }, ] -[[package]] -name = "nh3" -version = "0.3.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/86/f8d3a7c9bd1bbaa181f6312c757e0b74d25f71ecf84ea3c0dc5e0f01840d/nh3-0.3.4.tar.gz", hash = "sha256:96709a379997c1b28c8974146ca660b0dcd3794f4f6d50c1ea549bab39ac6ade", size = 19520, upload-time = "2026-03-25T10:57:30.789Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/5e/c400663d14be2216bc084ed2befc871b7b12563f85d40904f2a4bf0dd2b7/nh3-0.3.4-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8b61058f34c2105d44d2a4d4241bacf603a1ef5c143b08766bbd0cf23830118f", size = 1417991, upload-time = "2026-03-25T10:56:59.13Z" }, - { url = "https://files.pythonhosted.org/packages/36/f5/109526f5002ec41322ac8cafd50f0f154bae0c26b9607c0fcb708bdca8ec/nh3-0.3.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:554cc2bab281758e94d770c3fb0bf2d8be5fb403ef6b2e8841dd7c1615df7a0f", size = 790566, upload-time = "2026-03-25T10:57:00.445Z" }, - { url = "https://files.pythonhosted.org/packages/7b/66/38950f2b4b316ffd82ee51ed8f9143d1f56fdd620312cacc91613b77b3e7/nh3-0.3.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbe76feaa44e2ef9436f345016012a591550e77818876a8de5c8bc2a248e08df", size = 837538, upload-time = "2026-03-25T10:57:01.848Z" }, - { url = "https://files.pythonhosted.org/packages/d8/9f/9d6da970e9524fe360ea02a2082856390c2c8ba540409d1be6e5851887b3/nh3-0.3.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:87dac8d611b4a478400e0821a13b35770e88c266582f065e7249d6a37b0f86e8", size = 1012154, upload-time = "2026-03-25T10:57:03.592Z" }, - { url = "https://files.pythonhosted.org/packages/54/92/7c85c33c241e9dd51dda115bd3f765e940446588cdaaca62ef8edffe675f/nh3-0.3.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8d697e19f2995b337f648204848ac3a528eaafffc39e7ce4ac6b7a2fbe6c84af", size = 1092516, upload-time = "2026-03-25T10:57:04.726Z" }, - { url = "https://files.pythonhosted.org/packages/16/0f/597842bdb2890999a3faa2f3fcb02db8aa6ad09320d3d843ff6d0a1f737b/nh3-0.3.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:7cae217f031809321db962cd7e092bda8d4e95a87f78c0226628fa6c2ea8ebc5", size = 1053793, upload-time = "2026-03-25T10:57:06.171Z" }, - { url = "https://files.pythonhosted.org/packages/7d/32/669da65147bc10746d2e1d7a8a3dbfbffe0315f419e74b559e2ee3471a01/nh3-0.3.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:07999b998bf89692738f15c0eac76a416382932f855709e0b7488b595c30ec89", size = 1035975, upload-time = "2026-03-25T10:57:07.292Z" }, - { url = "https://files.pythonhosted.org/packages/a1/7e/9e97a8b3c5161c79b4bf21cc54e9334860a52cc54ede15bf2239ef494b73/nh3-0.3.4-cp314-cp314t-win32.whl", hash = "sha256:ca90397c8d36c1535bf1988b2bed006597337843a164c7ec269dc8813f37536b", size = 600419, upload-time = "2026-03-25T10:57:08.342Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c7/6849d8d4295d3997d148eacb2d4b1c9faada4895ee3c1b1e12e72f4611e2/nh3-0.3.4-cp314-cp314t-win_amd64.whl", hash = "sha256:41e46b3499918ab6128b6421677b316e79869d0c140da24069d220a94f4e72d1", size = 613342, upload-time = "2026-03-25T10:57:09.593Z" }, - { url = "https://files.pythonhosted.org/packages/8b/0e/14a3f510f36c20b922c123a2730f071f938d006fb513aacfd46d6cbc03a7/nh3-0.3.4-cp314-cp314t-win_arm64.whl", hash = "sha256:80b955d802bf365bd42e09f6c3d64567dce777d20e97968d94b3e9d9e99b265e", size = 607025, upload-time = "2026-03-25T10:57:10.959Z" }, - { url = "https://files.pythonhosted.org/packages/4a/57/a97955bc95960cfb1f0517043d60a121f4ba93fde252d4d9ffd3c2a9eead/nh3-0.3.4-cp38-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d8bebcb20ab4b91858385cd98fe58046ec4a624275b45ef9b976475604f45b49", size = 1439519, upload-time = "2026-03-25T10:57:12.019Z" }, - { url = "https://files.pythonhosted.org/packages/2b/60/c9a33361da8cde7c7760f091cd10467bc470634e4eea31c8bb70935b00a4/nh3-0.3.4-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d825722a1e8cbc87d7ca1e47ffb1d2a6cf343ad4c1b8465becf7cadcabcdfd0", size = 833798, upload-time = "2026-03-25T10:57:13.264Z" }, - { url = "https://files.pythonhosted.org/packages/6b/19/9487790780b8c94eacca37866c1270b747a4af8e244d43b3b550fddbbf62/nh3-0.3.4-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4aa8b43e68c26b68069a3b6cef09de166d1d7fa140cf8d77e409a46cbf742e44", size = 820414, upload-time = "2026-03-25T10:57:14.236Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b4/c6a340dd321d20b1e4a663307032741da045685c87403926c43656f6f5ec/nh3-0.3.4-cp38-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f5f214618ad5eff4f2a6b13a8d4da4d9e7f37c569d90a13fb9f0caaf7d04fe21", size = 1061531, upload-time = "2026-03-25T10:57:15.384Z" }, - { url = "https://files.pythonhosted.org/packages/c4/49/f6b4b474e0032e4bcbb7174b44e4cf6915670e09c62421deb06ccfcb88b8/nh3-0.3.4-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3390e4333883673a684ce16c1716b481e91782d6f56dec5c85fed9feedb23382", size = 1021889, upload-time = "2026-03-25T10:57:16.454Z" }, - { url = "https://files.pythonhosted.org/packages/43/da/e52a6941746d1f974752af3fc8591f1dbcdcf7fd8c726c7d99f444ba820e/nh3-0.3.4-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a2e44ccb29cbb45071b8f3f2dab9ebfb41a6516f328f91f1f1fd18196239a4", size = 912965, upload-time = "2026-03-25T10:57:17.624Z" }, - { url = "https://files.pythonhosted.org/packages/d6/b7/ec1cbc6b297a808c513f59f501656389623fc09ad6a58c640851289c7854/nh3-0.3.4-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0961a27dc2057c38d0364cb05880e1997ae1c80220cbc847db63213720b8f304", size = 804975, upload-time = "2026-03-25T10:57:18.994Z" }, - { url = "https://files.pythonhosted.org/packages/a9/56/b1275aa2c6510191eed76178da4626b0900402439cb9f27d6b9bf7c6d5e9/nh3-0.3.4-cp38-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:9337517edb7c10228252cce2898e20fb3d77e32ffaccbb3c66897927d74215a0", size = 833400, upload-time = "2026-03-25T10:57:20.086Z" }, - { url = "https://files.pythonhosted.org/packages/7c/a5/5d574ffa3c6e49a5364d1b25ebad165501c055340056671493beb467a15e/nh3-0.3.4-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d866701affe67a5171b916b5c076e767a74c6a9efb7fb2006eb8d3c5f9a293d5", size = 854277, upload-time = "2026-03-25T10:57:21.433Z" }, - { url = "https://files.pythonhosted.org/packages/79/36/8aeb2ab21517cefa212db109e41024e02650716cb42bf293d0a88437a92d/nh3-0.3.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:47d749d99ae005ab19517224140b280dd56e77b33afb82f9b600e106d0458003", size = 1022021, upload-time = "2026-03-25T10:57:22.433Z" }, - { url = "https://files.pythonhosted.org/packages/9c/95/9fd860997685e64abe2d5a995ca2eb5004c0fb6d6585429612a7871548b9/nh3-0.3.4-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f987cb56458323405e8e5ea827e1befcf141ffa0c0ac797d6d02e6b646056d9a", size = 1103526, upload-time = "2026-03-25T10:57:23.487Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0d/df545070614c1007f0109bb004230226c9000e7857c9785583ec25cda9d7/nh3-0.3.4-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:883d5a6d6ee8078c4afc8e96e022fe579c4c265775ff6ee21e39b8c542cabab3", size = 1068050, upload-time = "2026-03-25T10:57:24.624Z" }, - { url = "https://files.pythonhosted.org/packages/94/d5/17b016df52df052f714c53be71df26a1943551d9931e9383b92c998b88f8/nh3-0.3.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:75643c22f5092d8e209f766ee8108c400bc1e44760fc94d2d638eb138d18f853", size = 1046037, upload-time = "2026-03-25T10:57:25.799Z" }, - { url = "https://files.pythonhosted.org/packages/51/39/49f737907e6ab2b4ca71855d3bd63dd7958862e9c8b94fb4e5b18ccf6988/nh3-0.3.4-cp38-abi3-win32.whl", hash = "sha256:72e4e9ca1c4bd41b4a28b0190edc2e21e3f71496acd36a0162858e1a28db3d7e", size = 609542, upload-time = "2026-03-25T10:57:27.112Z" }, - { url = "https://files.pythonhosted.org/packages/73/4f/af8e9071d7464575a7316831938237ffc9d92d27f163dbdd964b1309cd9b/nh3-0.3.4-cp38-abi3-win_amd64.whl", hash = "sha256:c10b1f0c741e257a5cb2978d6bac86e7c784ab20572724b20c6402c2e24bce75", size = 624244, upload-time = "2026-03-25T10:57:28.302Z" }, - { url = "https://files.pythonhosted.org/packages/44/0c/37695d6b0168f6714b5c492331636a9e6123d6ec22d25876c68d06eab1b8/nh3-0.3.4-cp38-abi3-win_arm64.whl", hash = "sha256:43ad4eedee7e049b9069bc015b7b095d320ed6d167ecec111f877de1540656e9", size = 616649, upload-time = "2026-03-25T10:57:29.623Z" }, -] - [[package]] name = "openai" version = "2.54.0" @@ -1052,15 +706,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451, upload-time = "2024-11-08T09:47:44.722Z" }, ] -[[package]] -name = "pycparser" -version = "3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, -] - [[package]] name = "pydantic" version = "2.12.5" @@ -1194,15 +839,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] -[[package]] -name = "pygments" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, -] - [[package]] name = "pyproject-hooks" version = "1.2.0" @@ -1233,15 +869,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] -[[package]] -name = "pywin32-ctypes" -version = "0.2.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, -] - [[package]] name = "pyyaml" version = "6.0.3" @@ -1306,20 +933,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] -[[package]] -name = "readme-renderer" -version = "44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils" }, - { name = "nh3" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/a9/104ec9234c8448c4379768221ea6df01260cd6c2ce13182d4eac531c8342/readme_renderer-44.0.tar.gz", hash = "sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1", size = 32056, upload-time = "2024-07-08T15:00:57.805Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/67/921ec3024056483db83953ae8e48079ad62b92db7880013ca77632921dd0/readme_renderer-44.0-py3-none-any.whl", hash = "sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151", size = 13310, upload-time = "2024-07-08T15:00:56.577Z" }, -] - [[package]] name = "regex" version = "2026.9.29" @@ -1457,55 +1070,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/bc/97b2245c8c7b2dd01f2db74f2bea003cd33c15009b4996a2447f46b5325c/regex-2026.9.29-cp315-cp315t-win_arm64.whl", hash = "sha256:e8c65ef3862a8ad6e86492b6ed9327805dd66904c012bd3649dc67d822ed6c34", size = 285955, upload-time = "2026-09-29T00:49:55.655Z" }, ] -[[package]] -name = "requests" -version = "2.33.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, -] - -[[package]] -name = "requests-toolbelt" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, -] - -[[package]] -name = "rfc3986" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/40/1520d68bfa07ab5a6f065a186815fb6610c86fe957bc065754e47f7b0840/rfc3986-2.0.0.tar.gz", hash = "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c", size = 49026, upload-time = "2022-01-10T00:52:30.832Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/9a/9afaade874b2fa6c752c36f1548f718b5b83af81ed9b76628329dab81c1b/rfc3986-2.0.0-py2.py3-none-any.whl", hash = "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", size = 31326, upload-time = "2022-01-10T00:52:29.594Z" }, -] - -[[package]] -name = "rich" -version = "14.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, -] - [[package]] name = "ruff" version = "0.17.0" @@ -1531,19 +1095,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/71/eb4f0ccc844aece56963e8578df9c95d4c00f580547d52329d4035d3af18/ruff-0.17.0-py3-none-win_arm64.whl", hash = "sha256:c154c73ff43f9854395e24cac507af13078962e53d2b511605058d22af1fdb88", size = 9872035, upload-time = "2026-10-09T19:47:26.306Z" }, ] -[[package]] -name = "secretstorage" -version = "3.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, -] - [[package]] name = "six" version = "1.17.0" @@ -1641,26 +1192,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] -[[package]] -name = "twine" -version = "6.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "id" }, - { name = "keyring", marker = "platform_machine != 'ppc64le' and platform_machine != 's390x'" }, - { name = "packaging" }, - { name = "readme-renderer" }, - { name = "requests" }, - { name = "requests-toolbelt" }, - { name = "rfc3986" }, - { name = "rich" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e0/a8/949edebe3a82774c1ec34f637f5dd82d1cf22c25e963b7d63771083bbee5/twine-6.2.0.tar.gz", hash = "sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf", size = 172262, upload-time = "2025-09-04T15:43:17.255Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/7a/882d99539b19b1490cac5d77c67338d126e4122c8276bf640e411650c830/twine-6.2.0-py3-none-any.whl", hash = "sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8", size = 42727, upload-time = "2025-09-04T15:43:15.994Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" @@ -1682,15 +1213,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] -[[package]] -name = "urllib3" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, -] - [[package]] name = "uvicorn" version = "0.41.0" From 7fdbb3c26d932a8a2455aad64871179e6529ed10 Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:14:25 +0000 Subject: [PATCH 02/10] Let backends whose guests write host directories use CapFs A backend that mounts host directories into its guest needs the input directory's host path, as it already gets the output one; one that enforces the allowlist on raw sockets needs the permissions granted. CapFs::validate_output rescans the output directory after such a guest wrote it directly and applies the rules CapFs applies to writes made through it (limits, no symbolic or hard links, traversal bounds), plus no special files, which only a direct writer can make. CapFs::is_same_dir tells whether a path still names the directory CapFs opened, for a backend that hands the guest that path. Signed-off-by: danbugs --- src/hyperlight_sandbox/src/cap_fs.rs | 176 +++++++++++++++++++++++++- src/hyperlight_sandbox/src/network.rs | 8 ++ 2 files changed, 180 insertions(+), 4 deletions(-) diff --git a/src/hyperlight_sandbox/src/cap_fs.rs b/src/hyperlight_sandbox/src/cap_fs.rs index 3e66b68..e418210 100644 --- a/src/hyperlight_sandbox/src/cap_fs.rs +++ b/src/hyperlight_sandbox/src/cap_fs.rs @@ -228,6 +228,81 @@ impl Dir { } } +/// A directory's identity: device and inode on Unix, volume serial number +/// and 128-bit file id on Windows. +#[cfg(unix)] +type DirIdentity = (u64, u64); +#[cfg(windows)] +type DirIdentity = (u64, [u8; 16]); + +#[cfg(unix)] +fn dir_identity(dir: &CapDir) -> Result { + use cap_std::fs::MetadataExt; + + let meta = dir + .dir_metadata() + .context("failed to read directory metadata")?; + Ok((meta.dev(), meta.ino())) +} + +#[cfg(unix)] +fn path_identity(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + + let meta = std::fs::metadata(path) + .with_context(|| format!("failed to read metadata of {}", path.display()))?; + Ok((meta.dev(), meta.ino())) +} + +#[cfg(windows)] +fn dir_identity(dir: &CapDir) -> Result { + use std::os::windows::io::AsRawHandle; + + handle_identity(dir.as_raw_handle()) +} + +#[cfg(windows)] +fn path_identity(path: &Path) -> Result { + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; + + use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS; + + // Backup semantics are what lets a directory be opened as a file. + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path) + .with_context(|| format!("failed to open {}", path.display()))?; + handle_identity(file.as_raw_handle()) +} + +/// The 128-bit file id, unlike `GetFileInformationByHandle`'s 64-bit index, +/// is unique on ReFS too. +#[cfg(windows)] +fn handle_identity(handle: std::os::windows::io::RawHandle) -> Result { + use std::mem::MaybeUninit; + + use windows_sys::Win32::Storage::FileSystem::{ + FILE_ID_INFO, FileIdInfo, GetFileInformationByHandleEx, + }; + + let mut info = MaybeUninit::::zeroed(); + let ok = unsafe { + GetFileInformationByHandleEx( + handle as _, + FileIdInfo, + info.as_mut_ptr().cast(), + std::mem::size_of::() as u32, + ) + }; + if ok == 0 { + return Err(std::io::Error::last_os_error()).context("failed to read the file id"); + } + let info = unsafe { info.assume_init() }; + Ok((info.VolumeSerialNumber, info.FileId.Identifier)) +} + // --------------------------------------------------------------------------- // Internal types // --------------------------------------------------------------------------- @@ -283,6 +358,8 @@ pub struct CapFs { dir_streams: HashMap, next_handle: u32, + // Host filesystem path to the input directory. + input_path: Option, // Host filesystem path to the output directory. output_path: Option, // Owns the output temp dir when using the default. @@ -317,6 +394,7 @@ impl CapFs { streams: HashMap::new(), dir_streams: HashMap::new(), next_handle: FIRST_PREOPEN_FD, + input_path: None, output_path: None, _output_tmp: None, limits, @@ -356,6 +434,7 @@ impl CapFs { guest_path: "/input".to_string(), }, ); + self.input_path = Some(input_path.as_ref().to_path_buf()); Ok(self) } @@ -524,6 +603,14 @@ impl CapFs { Ok(data) } + /// Return the host filesystem path of the input directory, if configured. + /// + /// Backends that hand the guest a host directory directly (rather than + /// serving it through this `CapFs`) need the path to mount it. + pub fn input_path(&self) -> Option<&Path> { + self.input_path.as_deref() + } + /// Return the host filesystem path of the output directory, if configured. pub fn output_path(&self) -> Option<&Path> { self.output_path.as_deref() @@ -613,6 +700,19 @@ impl CapFs { self.preopen_dirs.contains_key(&fd) } + /// Whether `path` names the directory opened for `guest_path` (the same + /// file, not merely the same name): false once it was replaced, say. + /// + /// Backends that hand the guest a host directory by path, as a mount, + /// check it before doing so, since this `CapFs` checks and clears the + /// directory it opened. + pub fn is_same_dir(&self, guest_path: &str, path: &Path) -> Result { + let dir = self + .dir_by_guest_path(guest_path) + .ok_or_else(|| anyhow::anyhow!("no directory is open for {guest_path}"))?; + Ok(dir_identity(dir.cap_std())? == path_identity(path)?) + } + /// Find a preopened directory by its guest-visible path. pub fn dir_by_guest_path(&self, guest_path: &str) -> Option<&Dir> { self.preopen_dirs @@ -1220,7 +1320,24 @@ impl CapFs { } fn resynchronize_output(&mut self) -> Result<()> { - let usage = match self.scan_output_usage() { + self.resynchronize_output_with(false) + } + + /// Rescan the output directory after the guest wrote it directly, not + /// through this `CapFs`, and check it against the same rules: the + /// filesystem limits, no symbolic or hard links, the traversal depth + /// and entry bounds, and, since nothing else could have made one, no + /// special files (FIFOs, sockets, devices). The quota accounting is + /// brought up to date on success. + /// + /// For backends whose guest writes a host directory itself, such as + /// through a mount. + pub fn validate_output(&mut self) -> Result<()> { + self.resynchronize_output_with(true) + } + + fn resynchronize_output_with(&mut self, refuse_special: bool) -> Result<()> { + let usage = match self.scan_output_usage(refuse_special) { Ok(usage) => usage, Err(error) => { self.accounting_valid = false; @@ -1253,7 +1370,7 @@ impl CapFs { const MAX_SCAN_ENTRIES: usize = 100_000; const MAX_SCAN_DEPTH: usize = 64; - fn scan_output_usage(&self) -> Result { + fn scan_output_usage(&self, refuse_special: bool) -> Result { let Some(output_fd) = self.output_fd else { return Ok(FilesystemUsage::default()); }; @@ -1263,7 +1380,13 @@ impl CapFs { .ok_or_else(|| anyhow::anyhow!("output directory capability is missing"))?; let mut usage = FilesystemUsage::default(); let mut visited = 0usize; - self.scan_output_dir(output.dir.cap_std(), 0, &mut visited, &mut usage)?; + self.scan_output_dir( + output.dir.cap_std(), + 0, + &mut visited, + &mut usage, + refuse_special, + )?; Ok(usage) } @@ -1273,6 +1396,7 @@ impl CapFs { depth: usize, visited: &mut usize, usage: &mut FilesystemUsage, + refuse_special: bool, ) -> Result<()> { if depth > Self::MAX_SCAN_DEPTH { anyhow::bail!("output directory exceeds maximum traversal depth"); @@ -1296,7 +1420,7 @@ impl CapFs { let child = dir .open_dir(&name) .context("failed to open nested output directory")?; - self.scan_output_dir(&child, depth + 1, visited, usage)?; + self.scan_output_dir(&child, depth + 1, visited, usage, refuse_special)?; } else if metadata.is_file() { let metadata = Self::validated_output_metadata(dir, &name, metadata)?; if let Some(max_file_size) = self.limits.max_file_size @@ -1311,6 +1435,8 @@ impl CapFs { .ok_or(FsError::Overflow)?; usage.file_count = usage.file_count.checked_add(1).ok_or(FsError::Overflow)?; } + } else if refuse_special { + anyhow::bail!("special files are not supported in output directories"); } } Ok(()) @@ -1883,6 +2009,48 @@ mod tests { ); } + #[test] + fn is_same_dir_notices_a_replaced_directory() { + let (fs, output) = limited_output_fs(8, 32, 4); + assert!(fs.is_same_dir("/output", output.path()).unwrap()); + let moved = output.path().with_extension("moved"); + match std::fs::rename(output.path(), &moved) { + Ok(()) => {} + // Windows does not rename a directory this `CapFs` holds open, + // so it cannot be replaced at all there. + Err(_) if cfg!(windows) => return, + Err(error) => panic!("{error}"), + } + std::fs::create_dir(output.path()).unwrap(); + assert!(!fs.is_same_dir("/output", output.path()).unwrap()); + assert!(fs.is_same_dir("/output", &moved).unwrap()); + std::fs::remove_dir_all(&moved).unwrap(); + } + + #[test] + fn validate_output_checks_what_the_guest_wrote_directly() { + let (mut fs, output) = limited_output_fs(8, 32, 4); + fs.validate_output().unwrap(); + host_write_output(&output, "ok.bin", b"data"); + fs.validate_output().unwrap(); + host_write_output(&output, "big.bin", b"0123456789"); + assert!(fs.validate_output().is_err()); + } + + #[cfg(unix)] + #[test] + fn validate_output_refuses_special_files() { + let (mut fs, output) = limited_output_fs(8, 32, 4); + let made = std::process::Command::new("mkfifo") + .arg(output.path().join("pipe")) + .status() + .unwrap(); + assert!(made.success()); + assert!(fs.validate_output().is_err()); + // The scan for writes made through CapFs leaves them alone. + assert!(fs.resynchronize_output().is_ok()); + } + #[test] fn failed_create_does_not_consume_handle() { let (mut fs, output) = limited_output_fs(8, 32, 4); diff --git a/src/hyperlight_sandbox/src/network.rs b/src/hyperlight_sandbox/src/network.rs index b99a23b..56f6292 100644 --- a/src/hyperlight_sandbox/src/network.rs +++ b/src/hyperlight_sandbox/src/network.rs @@ -179,6 +179,14 @@ impl NetworkPermissions { Ok(()) } + /// The permissions granted so far, in the order they were added. + /// + /// Backends that enforce the allowlist below the HTTP layer (on raw + /// sockets, by host name) derive their own policy from these. + pub fn permissions(&self) -> &[NetworkPermission] { + &self.permissions + } + /// HTTP methods that are always blocked regardless of permissions. /// /// CONNECT enables HTTP tunneling and TRACE can reflect credentials. From e35594445ab764d8b77c553fcc7920a2b20174d8 Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:14:50 +0000 Subject: [PATCH 03/10] Add a hyperlight-unikraft sandbox backend hyperlight-unikraft-sandbox runs full CPython in a Unikraft micro-VM, from the published python, python-shell and agent rootfs images (the agent image adds numpy, pandas, scipy and scikit-learn), behind the same Guest / GuestSandbox contract as the Wasm and HyperlightJS backends. A prelude loaded at boot gives guest code call_tool, http_get and http_post as the Wasm Python guest has them, and the guest function every run calls. Requests and replies over the 64 KiB host call limit cross it in pieces, output in chunks bounded by their encoded size; the host holds at most 64 MiB of unread pieces per run. Exit statuses are a CPython process's, 0 to 255, leaving -1 to a failed guest. HTTP goes through the host allowlist; guest sockets are opt-in (native_network) and allowed by host name only, an unresolvable name being left out rather than failing every run. Snapshot and restore cover the whole micro-VM. /input and /output are hostfs mounts of paths resolved at build, and CapFs checks /output after each run (CapFs::validate_output): a run that breaks its rules fails. /output is cleared before each run and after a failed check, through the handle CapFs opened so that the working directory cannot redirect it, of every file and link and of the directories the guest made, however deep; the host's directories are kept. hostfs opens a mount when the guest boots, so before each boot the paths are checked to still be the directories CapFs opened: a replaced one fails the boot until it is put back. It builds against hyperlight-unikraft 0.18.0 and its images. That release's hyperlight-host 0.18 links beside the 0.17 hyperlight-wasm 0.15 and hyperlight-js 0.4 still need: hyperlight-sandbox shares no hyperlight type between backends. The guest's own output stays out of the host process (echo_output(false)), is capped like the rest, and its standard error, the kernel's messages included, goes to the run's stderr when written in whole lines. Child processes, threads and kills work in the guest as of 0.18, and the agent image carries the runtime marker hluk sizes it by. The end-to-end tests skip themselves when the images are not cached (just unikraft pull-images). Signed-off-by: danbugs --- .github/workflows/publish.yml | 1 + Cargo.lock | 167 ++- Cargo.toml | 6 + Justfile | 7 +- src/unikraft_sandbox/Cargo.toml | 26 + src/unikraft_sandbox/Justfile | 26 + src/unikraft_sandbox/examples/basics.rs | 132 +++ src/unikraft_sandbox/src/lib.rs | 990 ++++++++++++++++++ src/unikraft_sandbox/src/prelude.py | 295 ++++++ src/unikraft_sandbox/tests/python_image.rs | 640 +++++++++++ src/unikraft_sandbox/tests/relative_output.rs | 72 ++ 11 files changed, 2338 insertions(+), 24 deletions(-) create mode 100644 src/unikraft_sandbox/Cargo.toml create mode 100644 src/unikraft_sandbox/Justfile create mode 100644 src/unikraft_sandbox/examples/basics.rs create mode 100644 src/unikraft_sandbox/src/lib.rs create mode 100644 src/unikraft_sandbox/src/prelude.py create mode 100644 src/unikraft_sandbox/tests/python_image.rs create mode 100644 src/unikraft_sandbox/tests/relative_output.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2691f7a..27b77c7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -50,6 +50,7 @@ jobs: cargo publish --locked -p hyperlight-sandbox cargo publish --locked -p hyperlight-wasm-sandbox cargo publish --locked -p hyperlight-javascript-sandbox + cargo publish --locked -p hyperlight-unikraft-sandbox # Build all Python packages on Linux for glibc 2.28 and newer. build-linux: diff --git a/Cargo.lock b/Cargo.lock index f6e7aad..4c25188 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ "chrono", "git2", @@ -1527,15 +1527,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags 2.13.2", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -1769,6 +1768,26 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "hyperlight-common" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360d5876788b26af66bf055bca6b4f2c9c930a0fdcaebda4bd7f808c6bd93068" +dependencies = [ + "anyhow", + "bitflags 2.13.2", + "bytemuck", + "bytes", + "fixedbitset", + "flatbuffers", + "log", + "smallvec", + "spin 0.12.0", + "thiserror", + "tracing", + "tracing-core", +] + [[package]] name = "hyperlight-component-macro" version = "0.17.0" @@ -1810,7 +1829,7 @@ checksum = "816f673f3f8ef6d5703796bf92927b85177ad1baac6f391551a88451a9e46a7e" dependencies = [ "anyhow", "flatbuffers", - "hyperlight-common", + "hyperlight-common 0.17.0", "serde_json", "tracing", ] @@ -1823,7 +1842,7 @@ checksum = "9c2d0ea739be89822992706da9dc6812ce359b654b328efd6141ba1549751907" dependencies = [ "buddy_system_allocator", "flatbuffers", - "hyperlight-common", + "hyperlight-common 0.17.0", "hyperlight-guest", "hyperlight-guest-macro", "hyperlight-guest-tracing", @@ -1852,7 +1871,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f43a3d5d3fa245ce0fccee8a48ad7d94ec479e0898fc8f91fd440a1d65830c6" dependencies = [ - "hyperlight-common", + "hyperlight-common 0.17.0", "spin 0.12.0", "tracing", "tracing-core", @@ -1876,15 +1895,64 @@ dependencies = [ "flatbuffers", "goblin", "hex", - "hyperlight-common", + "hyperlight-common 0.17.0", "kvm-bindings", "kvm-ioctls", "lazy_static", "libc", "log", "metrics", - "mshv-bindings", - "mshv-ioctls", + "mshv-bindings 0.6.8", + "mshv-ioctls 0.6.8", + "oci-spec", + "page_size", + "parking_lot", + "rand 0.10.1", + "rust-embed", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "termcolor", + "thiserror", + "tracing", + "tracing-core", + "uuid", + "vmm-sys-util", + "windows", + "windows-result", + "windows-sys 0.61.2", + "windows-version", +] + +[[package]] +name = "hyperlight-host" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40f8b85ddd688890477158b9921b228973d1b70c84b866698838df0f32d2aa44" +dependencies = [ + "anyhow", + "bindgen", + "bitflags 2.13.2", + "blake3", + "built", + "bytemuck", + "cc", + "cfg-if", + "cfg_aliases", + "crossbeam-channel", + "flatbuffers", + "goblin", + "hex", + "hyperlight-common 0.18.0", + "kvm-bindings", + "kvm-ioctls", + "lazy_static", + "libc", + "log", + "metrics", + "mshv-bindings 0.7.1", + "mshv-ioctls 0.7.1", "oci-spec", "page_size", "parking_lot", @@ -1928,7 +1996,7 @@ dependencies = [ "anyhow", "cargo-hyperlight", "fn-traits", - "hyperlight-host", + "hyperlight-host 0.17.0", "hyperlight-js-common", "hyperlight-js-runtime", "metrics", @@ -1960,7 +2028,7 @@ dependencies = [ "hashbrown 0.17.0", "hex", "hmac", - "hyperlight-common", + "hyperlight-common 0.17.0", "hyperlight-guest", "hyperlight-guest-bin", "hyperlight-js-common", @@ -2055,6 +2123,39 @@ dependencies = [ "serde_json", ] +[[package]] +name = "hyperlight-unikraft" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f00b40d40daf67b908d9d6366b9bdb1f21b685179d24acfa4da64ac9d28bbc42" +dependencies = [ + "cap-std 4.0.3", + "getrandom 0.4.3", + "hyperlight-common 0.18.0", + "hyperlight-host 0.18.0", + "rustix 1.1.4", + "serde_json", + "sha2 0.11.0", + "thiserror", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "hyperlight-unikraft-sandbox" +version = "0.8.0" +dependencies = [ + "anyhow", + "base64", + "cap-std 4.0.3", + "hyperlight-sandbox", + "hyperlight-unikraft", + "serde", + "serde_json", + "tempfile", + "url", +] + [[package]] name = "hyperlight-wasm" version = "0.15.0" @@ -2070,8 +2171,8 @@ dependencies = [ "chrono", "env_logger", "goblin", - "hyperlight-common", - "hyperlight-host", + "hyperlight-common 0.17.0", + "hyperlight-host 0.17.0", "hyperlight-wasm-runtime", "junction", "libc", @@ -2109,7 +2210,7 @@ dependencies = [ "cc", "cfg-if", "cfg_aliases", - "hyperlight-common", + "hyperlight-common 0.17.0", "hyperlight-guest", "hyperlight-guest-bin", "hyperlight-wasm-macro", @@ -2128,9 +2229,9 @@ dependencies = [ "http", "http-body", "http-body-util", - "hyperlight-common", + "hyperlight-common 0.17.0", "hyperlight-component-macro", - "hyperlight-host", + "hyperlight-host 0.17.0", "hyperlight-sandbox", "hyperlight-wasm", "log", @@ -2491,9 +2592,9 @@ checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.8+1.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "7f7c568b25d7489bc3fb2988ed69ab111d2944d2f5fec3d5c987fe545ea97b50" dependencies = [ "cc", "libc", @@ -2663,6 +2764,18 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "mshv-bindings" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4b2414499597c6e31b1cd25239f19e8e8c55023b3bf8fcfa44d927cff6a123" +dependencies = [ + "libc", + "num_enum", + "vmm-sys-util", + "zerocopy", +] + [[package]] name = "mshv-ioctls" version = "0.6.8" @@ -2670,7 +2783,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1339723fe3a26baf4041459de20ad923e89d312c3bb25dbf9f60738c22a47f5e" dependencies = [ "libc", - "mshv-bindings", + "mshv-bindings 0.6.8", + "thiserror", + "vmm-sys-util", +] + +[[package]] +name = "mshv-ioctls" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30d1b5b76a709e30b46248f52bbb44dde04a659d6d58255c7f06c88d528d210" +dependencies = [ + "libc", + "mshv-bindings 0.7.1", "thiserror", "vmm-sys-util", ] diff --git a/Cargo.toml b/Cargo.toml index 1cf04cf..2a7fa26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "src/hyperlight_sandbox", "src/javascript_sandbox", "src/wasm_sandbox", + "src/unikraft_sandbox", "src/sdk/python/pyo3_common", "src/sdk/python/wasm_backend", "src/sdk/python/hyperlight_js_backend", @@ -23,9 +24,14 @@ categories = ["virtualization"] hyperlight-sandbox = { version = "0.8.0", path = "src/hyperlight_sandbox" } hyperlight-javascript-sandbox = { version = "0.8.0", path = "src/javascript_sandbox" } hyperlight-wasm-sandbox = { version = "0.8.0", path = "src/wasm_sandbox" } +hyperlight-unikraft-sandbox = { version = "0.8.0", path = "src/unikraft_sandbox" } hyperlight-sandbox-pyo3-common = { path = "src/sdk/python/pyo3_common" } hyperlight-common = { version = "0.17.0", default-features = false } hyperlight-component-macro = "0.17.0" hyperlight-host = { version = "0.17.0", default-features = false, features = ["executable_heap"] } hyperlight-wasm = "0.15.0" +# Brings its own hyperlight-host/-common 0.18, apart from the 0.17 above that +# hyperlight-wasm and hyperlight-js need: no hyperlight type crosses between +# the backends, so the two link side by side. +hyperlight-unikraft = { version = "0.18.0", default-features = false } pyo3 = { version = "0.29", features = ["extension-module"] } diff --git a/Justfile b/Justfile index bdad65f..e78cfaf 100644 --- a/Justfile +++ b/Justfile @@ -2,6 +2,7 @@ set unstable := true mod wasm 'src/wasm_sandbox/Justfile' mod js 'src/javascript_sandbox/Justfile' +mod unikraft 'src/unikraft_sandbox/Justfile' mod python 'src/sdk/python/Justfile' mod dotnet 'src/sdk/dotnet/Justfile' mod examples_mod 'examples/Justfile' @@ -13,9 +14,9 @@ clean: wasm::clean python::clean dotnet::clean #### BUILD TARGETS #### -build target=default-target: (wasm::build target) (js::build target) python::build (dotnet::build target) +build target=default-target: (wasm::build target) (js::build target) (unikraft::build target) python::build (dotnet::build target) -lint: lint-rust wasm::lint js::lint python::lint +lint: lint-rust wasm::lint js::lint unikraft::lint python::lint lint-rust: cargo clippy -p hyperlight-sandbox --all-targets --features test-utils -- -D warnings @@ -32,7 +33,7 @@ fmt-check-rust: #### TESTS #### -test: wasm::guest-build wasm::js-guest-build python::build python::python-test test-rust wasm::test dotnet::test-rust dotnet::test +test: wasm::guest-build wasm::js-guest-build python::build python::python-test test-rust wasm::test unikraft::test dotnet::test-rust dotnet::test fuzz seconds="60": (python::python-fuzz seconds) diff --git a/src/unikraft_sandbox/Cargo.toml b/src/unikraft_sandbox/Cargo.toml new file mode 100644 index 0000000..5acfc1f --- /dev/null +++ b/src/unikraft_sandbox/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "hyperlight-unikraft-sandbox" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +description = "hyperlight-unikraft guest for hyperlight-sandbox: full CPython in a Unikraft micro-VM" + +[dependencies] +hyperlight-sandbox.workspace = true +# The library only: the default `cli` feature pulls in the `hluk` CLI's +# registry client, TOML and archive dependencies. +hyperlight-unikraft = { workspace = true } +anyhow = "1" +base64 = "0.22" +cap-std = "4.0.3" +serde_json = "1" +url = "2" + +[dev-dependencies] +hyperlight-sandbox = { workspace = true, features = ["test-utils"] } +serde = { version = "1", features = ["derive"] } +tempfile = "3" diff --git a/src/unikraft_sandbox/Justfile b/src/unikraft_sandbox/Justfile new file mode 100644 index 0000000..be1b7fc --- /dev/null +++ b/src/unikraft_sandbox/Justfile @@ -0,0 +1,26 @@ +set windows-shell := ["pwsh", "-NoLogo", "-Command"] +default-target := "debug" +repo-root := invocation_directory_native() + +build target=default-target: + cargo build --manifest-path {{repo-root}}/src/unikraft_sandbox/Cargo.toml --profile={{ if target == "debug" {"dev"} else { target } }} + +# Pull the published rootfs images the examples and tests run on into the +# hluk CLI's cache (needs `hluk`: https://github.com/hyperlight-dev/hyperlight-unikraft). +pull-images: + hluk run --runtime python --cold --exec "pass" + hluk run --runtime agent --cold --exec "pass" + +#### EXAMPLES #### + +examples target=default-target: + cargo run --manifest-path {{repo-root}}/src/unikraft_sandbox/Cargo.toml --profile={{ if target == "debug" {"dev"} else { target } }} --example basics + +#### TESTS #### + +lint: + cargo clippy -p hyperlight-unikraft-sandbox --all-targets -- -D warnings + +# The end-to-end tests skip themselves when the images are not cached. +test: + cargo test -p hyperlight-unikraft-sandbox diff --git a/src/unikraft_sandbox/examples/basics.rs b/src/unikraft_sandbox/examples/basics.rs new file mode 100644 index 0000000..25942cd --- /dev/null +++ b/src/unikraft_sandbox/examples/basics.rs @@ -0,0 +1,132 @@ +//! Basics demo for the hyperlight-unikraft sandbox backend. +//! +//! Exercises: CPython with numpy/pandas, state that persists across runs, +//! tool dispatch, /output files, errors, and snapshot/restore, with timings. +//! +//! Needs the `agent` rootfs. Pass its path as the first argument, or let +//! the example find the copy the `hluk` CLI caches: +//! +//! ```sh +//! cargo run --release -p hyperlight-unikraft-sandbox --example basics [ROOTFS.cpio] +//! ``` + +use std::time::Instant; + +use hyperlight_sandbox::{SandboxBuilder, ToolRegistry}; +use hyperlight_unikraft_sandbox::{Unikraft, cached_rootfs}; +use serde::Deserialize; + +fn separator(title: &str) { + println!("\n{}", "═".repeat(60)); + println!("{title}"); + println!("{}", "═".repeat(60)); +} + +#[derive(Deserialize)] +struct MathArgs { + a: f64, + b: f64, +} + +#[derive(Deserialize)] +struct FetchArgs { + table: String, +} + +fn main() -> anyhow::Result<()> { + let rootfs = match std::env::args().nth(1) { + Some(path) => path.into(), + None => cached_rootfs("agent")?, + }; + + let mut tools = ToolRegistry::new(); + tools.register_typed::("multiply", |args| Ok(serde_json::json!(args.a * args.b))); + tools.register_typed::("fetch_data", |args| match args.table.as_str() { + "sales" => Ok(serde_json::json!([ + {"region": "north", "units": 120, "price": 9.5}, + {"region": "south", "units": 80, "price": 11.0}, + {"region": "north", "units": 45, "price": 9.5}, + {"region": "east", "units": 200, "price": 7.25}, + ])), + other => anyhow::bail!("no table named {other:?}"), + }); + + separator("Boot"); + let started = Instant::now(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs.display().to_string()) + .with_tools(tools) + .temp_output() + .guest(Unikraft::new()) + .build()?; + println!("booted {} in {:?}", rootfs.display(), started.elapsed()); + + separator("Test 1: numpy + pandas"); + let started = Instant::now(); + let result = sandbox.run( + r#" +import sys, numpy as np, pandas as pd +print(f"Python {sys.version.split()[0]}, numpy {np.__version__}, pandas {pd.__version__}") +a = np.arange(1, 10).reshape(3, 3) +print("det(a + I) =", round(float(np.linalg.det(a + np.eye(3))), 3)) +"#, + )?; + print!("{}", result.stdout); + eprint!("{}", result.stderr); + println!("({:?})", started.elapsed()); + assert_eq!(result.exit_code, 0); + assert!(result.stdout.contains("numpy")); + + separator("Test 2: state persists across runs"); + sandbox.run("counter = 41")?; + let started = Instant::now(); + let result = sandbox.run("counter += 1\nprint('counter =', counter)")?; + print!("{}", result.stdout); + println!("({:?})", started.elapsed()); + assert!(result.stdout.contains("counter = 42")); + + separator("Test 3: tools + /output"); + let started = Instant::now(); + let result = sandbox.run( + r#" +import pandas as pd +df = pd.DataFrame(call_tool('fetch_data', table='sales')) +df["revenue"] = df["units"] * df["price"] +summary = df.groupby("region")["revenue"].sum().sort_values(ascending=False) +print(summary.to_string()) +print("6 x 7 =", call_tool('multiply', a=6, b=7)) +summary.to_csv("/output/summary.csv") +try: + call_tool('fetch_data', table='nope') +except RuntimeError as e: + print("tool error surfaced:", e) +"#, + )?; + print!("{}", result.stdout); + eprint!("{}", result.stderr); + println!("({:?})", started.elapsed()); + assert_eq!(result.exit_code, 0); + assert!(result.stdout.contains("6 x 7 = 42")); + println!("output files: {:?}", sandbox.get_output_files()?); + + separator("Test 4: errors land in stderr with an exit code"); + let result = sandbox.run("1 / 0")?; + print!("{}", result.stderr); + assert_eq!(result.exit_code, 1); + assert!(result.stderr.contains("ZeroDivisionError")); + + separator("Test 5: snapshot / restore"); + let started = Instant::now(); + let snapshot = sandbox.snapshot()?; + println!("snapshot: {:?}", started.elapsed()); + sandbox.run("counter = 100\nimport sklearn")?; + let started = Instant::now(); + sandbox.restore(&snapshot)?; + println!("restore: {:?}", started.elapsed()); + let result = sandbox.run("print('counter =', counter)")?; + print!("{}", result.stdout); + assert!(result.stdout.contains("counter = 42")); + + separator("All tests passed!"); + Ok(()) +} diff --git a/src/unikraft_sandbox/src/lib.rs b/src/unikraft_sandbox/src/lib.rs new file mode 100644 index 0000000..31f0a99 --- /dev/null +++ b/src/unikraft_sandbox/src/lib.rs @@ -0,0 +1,990 @@ +//! [hyperlight-unikraft] guest for hyperlight-sandbox. +//! +//! Runs a full CPython (the published `python`, `python-shell` and `agent` +//! images: the `agent` image adds numpy, pandas, scipy, scikit-learn and +//! friends) in a Unikraft unikernel inside a Hyperlight micro-VM, behind the +//! same [`Guest`] / [`GuestSandbox`] contract as the Wasm and HyperlightJS +//! backends: +//! +//! | hyperlight-sandbox | hyperlight-unikraft | +//! |--------------------------------|----------------------------------------------| +//! | `run(code)` | a guest function call into a boot-time bridge | +//! | `call_tool(name, **kwargs)` | host function `sandbox.call_tool` | +//! | `http_get` / `http_post` | host function `sandbox.http` (allowlisted) | +//! | `/input`, `/output` | hostfs mounts (read-only / read-write) | +//! | `allow_domain` | checked per HTTP request on the host; raw sockets opt-in via [`Unikraft::native_network`] | +//! | `snapshot` / `restore` | hyperlight snapshots of the whole micro-VM | +//! +//! The module path given to the sandbox builder is the rootfs image (an +//! initrd CPIO). [`cached_rootfs`] finds one the `hluk` CLI has pulled. +//! +//! [hyperlight-unikraft]: https://github.com/hyperlight-dev/hyperlight-unikraft + +use std::borrow::Cow; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use anyhow::{Context, Result}; +use base64::Engine as _; +use cap_std::fs::Dir as CapDir; +use hyperlight_sandbox::runtime::BlockOn; +use hyperlight_sandbox::{ + CapFs, DirPerms, ExecutionResult, FilePerms, Guest, GuestSandbox, HttpMethod, + NetworkPermissions, SandboxConfig, Snapshot, ToolRegistry, http as sandbox_http, +}; +pub use hyperlight_unikraft; +use hyperlight_unikraft::{ + AllowList, AppSandbox, Mount, NetworkPolicy, SandboxBuilder as HlukBuilder, + Snapshot as HlukSnapshot, +}; + +/// The guest code that defines `call_tool`, `http_get`, `http_post` and the +/// run bridge. Run once in `__main__` at boot, so snapshots carry it. +const PRELUDE: &str = include_str!("prelude.py"); + +/// The guest function the prelude defines; every `run` calls it. +const RUN_FUNCTION: &str = "__hyperlight_sandbox_run"; + +/// Largest reply sent inline in one host call. The limit is 64 KiB for the +/// whole reply, framing included; anything larger goes through the blob +/// store and the guest pulls it in [`BLOB_CHUNK`] pieces. +const INLINE_MAX: usize = 48 * 1024; + +/// Raw bytes per `sandbox.blob_read` reply: base64 grows it by 4/3, to +/// ~43 KiB, safely under the 64 KiB host call limit. +const BLOB_CHUNK: usize = 32 * 1024; + +/// Cap on captured stdout and stderr per run, matching the JS backend. +const MAX_OUTPUT_BYTES: usize = 16 * 1024 * 1024; + +/// Cap on the bytes the host holds for the guest at once: replies it has +/// not finished reading and requests it is still uploading. Without it a +/// guest could pin host memory by making large calls and never reading +/// the replies. +const MAX_PENDING_BYTES: usize = 64 * 1024 * 1024; + +/// Cap on the number of replies and uploads held at once. +const MAX_PENDING_BLOBS: usize = 256; + +/// The release of the published rootfs images this backend is built +/// against. A snapshot (and the guest kernel's host contract) is tied to +/// the `hyperlight-unikraft` release, so the images must match it. +pub const IMAGE_RELEASE: &str = "v0.18.0"; + +/// The path where the `hluk` CLI caches the published rootfs image of +/// `runtime` (`python`, `python-shell`, `agent`, ...) for [`IMAGE_RELEASE`]. +/// +/// Mirrors the CLI's layout: `$HLUK_CACHE_DIR`, else `$XDG_CACHE_HOME/hluk` +/// or `~/.cache/hluk` on Unix, `%LOCALAPPDATA%\hluk\cache` on Windows. +/// Errors if the image has not been pulled; any `hluk run` of the image +/// pulls it. +pub fn cached_rootfs(runtime: &str) -> Result { + let cache = if let Some(dir) = std::env::var_os("HLUK_CACHE_DIR") { + PathBuf::from(dir) + } else if cfg!(windows) { + std::env::var_os("LOCALAPPDATA") + .map(|d| PathBuf::from(d).join("hluk").join("cache")) + .context("cannot find the hluk cache: set HLUK_CACHE_DIR or LOCALAPPDATA")? + } else if let Some(dir) = std::env::var_os("XDG_CACHE_HOME").filter(|d| !d.is_empty()) { + PathBuf::from(dir).join("hluk") + } else { + std::env::var_os("HOME") + .map(|h| PathBuf::from(h).join(".cache").join("hluk")) + .context("cannot find the hluk cache: set HLUK_CACHE_DIR or HOME")? + }; + let path = cache + .join("rootfs/ghcr.io/hyperlight-dev/hyperlight-unikraft") + .join(runtime) + .join(format!("initrd-{IMAGE_RELEASE}.cpio")); + anyhow::ensure!( + path.is_file(), + "no cached {runtime} rootfs at {}; pull it with `hluk run --runtime {runtime} --cold --exec pass` \ + (hluk {IMAGE_RELEASE}) or pass the CPIO path as the module path", + path.display() + ); + Ok(path) +} + +// --------------------------------------------------------------------------- +// Guest +// --------------------------------------------------------------------------- + +/// The hyperlight-unikraft guest. The sandbox's module path is the rootfs. +/// +/// `SandboxConfig::heap_size` / `stack_size` do not apply: a unikernel has +/// one guest memory size, which defaults to the size the image was tested +/// with (1.5 GiB for `agent`, 256 MiB for `python`) and is set with +/// [`scratch_mb`](Self::scratch_mb). +/// +/// `/input` and `/output` are host directories mounted at boot. `/output` +/// is read-write when its permissions allow creating and writing files, +/// else read-only when they allow reading, else absent; it cannot be +/// write-only. As in the other backends, its files persist from run to +/// run, and restoring a snapshot clears them. Files the guest has open are +/// interpreter state, though: they stay open across runs, and a restore +/// does not close them. Since the guest writes `/output` directly, it is +/// checked by the rules `CapFs` applies to writes made through it (the +/// filesystem limits, no symbolic or hard links, no special files, at most +/// 64 levels and 100,000 entries) after each run, and before each, as the +/// host may have changed it since. A run that breaks them fails, and +/// `/output` is cleared of every file and link and of the directories made +/// during the run; those it had when the run started are kept, emptied. A +/// run that would start on such things is refused, and they are left as +/// they are. The guest is (re)booted +/// only onto the directories the sandbox was built with: one replaced on +/// the host since then fails the boot an allowlist change makes, until it +/// is put back. +/// +/// A run's exit status is the one a CPython process would exit with, 0 to +/// 255 (`sys.exit(-1)` is 255), so -1 always means the guest failed. +/// +/// Requests and replies of any size cross the 64 KiB host call limit in +/// pieces; the host holds at most 64 MiB (and 256 pieces) of them unread +/// at a time, and a call past that fails in the guest. +#[derive(Debug, Clone, Copy, Default)] +pub struct Unikraft { + scratch_mb: Option, + native_network: bool, +} + +impl Unikraft { + pub fn new() -> Self { + Self::default() + } + + /// Guest memory in MiB, overriding the image's tested default. + pub fn scratch_mb(mut self, mb: usize) -> Self { + self.scratch_mb = Some(mb); + self + } + + /// Also let guest sockets reach the hosts named by `allow_domain`, so + /// libraries like `requests` and `urllib` work directly. + /// + /// Off by default because socket-level enforcement is coarser than the + /// HTTP-level allowlist the other backends apply: it is by host name + /// only, so the scheme, port, path prefix and method filters of an + /// `allow_domain` entry are not enforced on these sockets (they still + /// are for `http_get` / `http_post`). The names are resolved when the + /// allowlist changes; one that does not resolve then is left out until + /// it changes again, and the next run's stderr says so. + pub fn native_network(mut self, on: bool) -> Self { + self.native_network = on; + self + } +} + +impl Guest for Unikraft { + type Sandbox = UnikraftGuestSandbox; + + fn build( + self, + config: SandboxConfig, + tools: ToolRegistry, + network: Arc>, + fs: Arc>, + ) -> Result { + UnikraftGuestSandbox::new(self, config, tools, network, fs) + } +} + +// --------------------------------------------------------------------------- +// Host state shared with the host functions +// --------------------------------------------------------------------------- + +/// Data too large for one host call: replies waiting for the guest to read +/// them, and requests the guest is uploading. Emptied before each run. +#[derive(Default)] +struct Blobs { + next_id: u64, + pending: HashMap>, + /// Bytes held across `pending`. + bytes: usize, +} + +impl Blobs { + /// Make room for `extra` more bytes, in a new blob if `new`. + fn reserve(&self, extra: usize, new: bool) -> std::result::Result<(), String> { + if new && self.pending.len() >= MAX_PENDING_BLOBS { + return Err(format!( + "the host holds {MAX_PENDING_BLOBS} unread replies or uploads already; \ + read them before making more calls" + )); + } + if self.bytes.saturating_add(extra) > MAX_PENDING_BYTES { + return Err(format!( + "the host would hold more than {MAX_PENDING_BYTES} bytes of unread replies \ + and uploads; read earlier replies, or exchange less data per call" + )); + } + Ok(()) + } + + fn insert(&mut self, data: Vec) -> std::result::Result { + self.reserve(data.len(), true)?; + let id = self.next_id; + self.next_id += 1; + self.bytes += data.len(); + self.pending.insert(id, data); + Ok(id) + } + + fn remove(&mut self, id: u64) -> Option> { + let data = self.pending.remove(&id)?; + self.bytes -= data.len(); + Some(data) + } + + fn clear(&mut self) { + self.pending.clear(); + self.bytes = 0; + } +} + +/// The upload a host call's arguments name, if they are `[{"upload": id}]`: +/// a request the guest sent in pieces, being too large for one host call. +/// No call's own arguments take that shape: they have two or four elements. +fn uploaded(args: &str) -> Option { + // json.dumps spells it so; skip parsing every other call twice. + if !args.starts_with("[{\"upload\"") { + return None; + } + let serde_json::Value::Array(items) = serde_json::from_str(args).ok()? else { + return None; + }; + let [serde_json::Value::Object(fields)] = items.as_slice() else { + return None; + }; + if fields.len() != 1 { + return None; + } + fields.get("upload")?.as_u64() +} + +#[derive(Clone)] +struct HostState { + tools: Arc, + network: Arc>, + blobs: Arc>, + /// What Python writes to `sys.stderr`, apart from standard output. + /// Python's standard output is the guest's fd 1, with what everything + /// else writes there, in order. + stderr: Arc>, +} + +impl HostState { + /// Wrap a reply value in the envelope the prelude's `_hls_unwrap` reads. + fn envelope(&self, value: &serde_json::Value) -> std::result::Result { + let text = serde_json::to_string(value).map_err(|e| e.to_string())?; + if text.len() <= INLINE_MAX { + return Ok(format!("{{\"inline\":{text}}}")); + } + let mut blobs = self.blobs.lock().map_err(|_| "blob store poisoned")?; + let len = text.len(); + let id = blobs.insert(text.into_bytes())?; + Ok(serde_json::json!({ "blob": id, "len": len }).to_string()) + } + + /// Append a piece of an upload, `[id or null, text]`, starting a new + /// one for `null`; returns the upload's id. + fn upload(&self, args: &str) -> std::result::Result { + let (id, piece): (Option, String) = + serde_json::from_str(args).map_err(|e| e.to_string())?; + let mut blobs = self.blobs.lock().map_err(|_| "blob store poisoned")?; + let id = match id { + None => blobs.insert(piece.into_bytes())?, + Some(id) => { + blobs.reserve(piece.len(), false)?; + let data = blobs + .pending + .get_mut(&id) + .ok_or_else(|| format!("no upload {id}"))?; + data.extend_from_slice(piece.as_bytes()); + blobs.bytes += piece.len(); + id + } + }; + Ok(id.to_string()) + } + + /// The arguments of a host call, from an upload if the guest sent one. + fn arguments<'a>(&self, args: &'a str) -> std::result::Result, String> { + let Some(upload) = uploaded(args) else { + return Ok(Cow::Borrowed(args)); + }; + let data = self + .blobs + .lock() + .map_err(|_| "blob store poisoned")? + .remove(upload) + .ok_or_else(|| format!("no upload {upload}"))?; + String::from_utf8(data) + .map(Cow::Owned) + .map_err(|_| "upload is not UTF-8".to_string()) + } + + fn blob_read(&self, args: &str) -> std::result::Result { + let (id, offset): (u64, usize) = serde_json::from_str(args).map_err(|e| e.to_string())?; + let mut blobs = self.blobs.lock().map_err(|_| "blob store poisoned")?; + let blob = blobs + .pending + .get(&id) + .ok_or_else(|| format!("no host blob {id}"))?; + let end = blob.len().min(offset.saturating_add(BLOB_CHUNK)); + let chunk = blob.get(offset..end).unwrap_or_default(); + let encoded = base64::engine::general_purpose::STANDARD.encode(chunk); + // The guest reads front to back, so the last chunk frees the blob. + if end >= blob.len() { + blobs.remove(id); + } + serde_json::to_string(&encoded).map_err(|e| e.to_string()) + } + + fn call_tool(&self, args: &str) -> std::result::Result { + let args = self.arguments(args)?; + let (name, kwargs): (String, serde_json::Value) = + serde_json::from_str(&args).map_err(|e| e.to_string())?; + let value = self + .tools + .dispatch(&name, kwargs) + .map_err(|e| format!("{e:#}"))?; + self.envelope(&value) + } + + fn http(&self, args: &str) -> std::result::Result { + let args = self.arguments(args)?; + let (method, url, headers, body): ( + String, + String, + HashMap, + Option, + ) = serde_json::from_str(&args).map_err(|e| e.to_string())?; + let url = url::Url::parse(&url).map_err(|e| format!("invalid URL: {e}"))?; + let method_enum: HttpMethod = method.parse().map_err(|e| format!("{e}"))?; + { + let network = self.network.lock().map_err(|_| "network mutex poisoned")?; + if !network.is_allowed(&url, &method_enum) { + return Err(format!("HTTP request denied for {method_enum} {url}")); + } + } + if headers.len() > sandbox_http::MAX_RESPONSE_HEADER_COUNT { + return Err("too many request headers".into()); + } + let request = sandbox_http::HttpRequest { + url, + method: method_enum.to_string(), + headers: headers.into_iter().collect(), + body: sandbox_http::HttpRequest::body_from_bytes(body.map(String::into_bytes)), + }; + let response = sandbox_http::send_http_request(request) + .block_on() + .map_err(|e| format!("{e:#}"))?; + let body = match String::from_utf8(response.body) { + Ok(body) => body, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + }; + self.envelope(&serde_json::json!({ + "status": response.status, + "headers": response.headers, + "body": body, + })) + } + + fn append(buffer: &Mutex, args: &str) -> std::result::Result { + let [text]: [String; 1] = serde_json::from_str(args).map_err(|e| e.to_string())?; + let mut buffer = buffer.lock().map_err(|_| "output buffer poisoned")?; + push_capped(&mut buffer, &text); + Ok(String::new()) + } + + /// Register every bridge host function on a hyperlight-unikraft builder. + fn register(&self, builder: HlukBuilder) -> HlukBuilder { + let s = self.clone(); + let builder = builder.host_function("sandbox.call_tool", move |a| s.call_tool(a)); + let s = self.clone(); + let builder = builder.host_function("sandbox.http", move |a| s.http(a)); + let s = self.clone(); + let builder = builder.host_function("sandbox.blob_read", move |a| s.blob_read(a)); + let s = self.clone(); + let builder = builder.host_function("sandbox.upload", move |a| s.upload(a)); + let err = self.stderr.clone(); + builder.host_function("sandbox.stderr", move |a| Self::append(&err, a)) + } + + fn take_stderr(&self) -> Result { + self.stderr + .lock() + .map(|mut b| std::mem::take(&mut *b)) + .map_err(|_| anyhow::anyhow!("output buffer poisoned")) + } +} + +/// Append `text`, cut at a character boundary once the buffer is full. +fn push_capped(buffer: &mut String, text: &str) { + let room = MAX_OUTPUT_BYTES.saturating_sub(buffer.len()); + let mut take = text.len().min(room); + while !text.is_char_boundary(take) { + take -= 1; + } + buffer.push_str(&text[..take]); +} + +// --------------------------------------------------------------------------- +// Sandbox +// --------------------------------------------------------------------------- + +pub struct UnikraftGuestSandbox { + sandbox: AppSandbox, + host: HostState, + mounts: Vec, + native_network: bool, + /// The hosts the guest's socket policy was built from, to notice an + /// `allow_domain` made after boot. + socket_hosts: Vec, + /// Said in the next run's stderr: which allowed hosts guest sockets + /// cannot reach because they did not resolve. + socket_notice: Option, + files: Arc>, + /// The directories `/output` had when the current run started, relative + /// to it: kept when a failed check clears the run's output, so that + /// what the host staged keeps its shape. + kept_dirs: HashSet, +} + +impl UnikraftGuestSandbox { + fn new( + guest: Unikraft, + config: SandboxConfig, + tools: ToolRegistry, + network: Arc>, + files: Arc>, + ) -> Result { + // HTTP host calls run on the shared runtime; fail now, not mid-run. + hyperlight_sandbox::runtime::RUNTIME + .as_ref() + .map_err(|e| anyhow::anyhow!("{e}"))?; + anyhow::ensure!( + !config.module_path.is_empty(), + "the unikraft backend needs a rootfs image as its module path \ + (an initrd CPIO, e.g. from hyperlight_unikraft_sandbox::cached_rootfs(\"agent\"))" + ); + let rootfs = PathBuf::from(&config.module_path); + anyhow::ensure!( + rootfs.is_file(), + "rootfs image does not exist: {}", + rootfs.display() + ); + + let host = HostState { + tools: Arc::new(tools), + network: network.clone(), + blobs: Arc::default(), + stderr: Arc::default(), + }; + let mounts = mounts_for(&files)?; + let socket_hosts = socket_hosts(guest.native_network, &network)?; + let (policy, socket_notice) = socket_policy(&socket_hosts); + + let mut builder = HlukBuilder::from_initrd(&rootfs); + if let Some(mb) = guest.scratch_mb { + builder = builder.scratch_mb(mb); + } + check_mounts(&files, &mounts)?; + let builder = configure(builder, &host, &mounts, policy); + let mut sandbox = builder + .boot() + .with_context(|| format!("failed to boot {}", rootfs.display()))?; + anyhow::ensure!( + sandbox.has_driver(), + "{} has no runtime driver to run code with; use a python, python-shell or agent image", + rootfs.display() + ); + sandbox.run(PRELUDE).map_err(|e| { + anyhow::anyhow!( + "failed to load the sandbox bridge into {} (only the python, python-shell and \ + agent images are supported): {e}\n{}", + rootfs.display(), + sandbox.drain_output() + ) + })?; + sandbox.drain_output(); + + Ok(Self { + sandbox, + host, + mounts, + native_network: guest.native_network, + socket_hosts, + socket_notice, + files, + // Taken before each run + kept_dirs: HashSet::new(), + }) + } + + /// Re-boot the guest under the current allowlist, keeping its state: + /// the socket policy is fixed at boot, and a snapshot restores under a + /// different one. + fn sync_socket_policy(&mut self) -> Result<()> { + let hosts = socket_hosts(self.native_network, &self.host.network)?; + if hosts == self.socket_hosts { + return Ok(()); + } + // Resolved once per change of the allowlist, not every run: a name + // that does not resolve leaves guest sockets unable to reach it, as + // DNS would, rather than failing every run from then on. + let (policy, notice) = socket_policy(&hosts); + check_mounts(&self.files, &self.mounts)?; + let snapshot = self.sandbox.snapshot()?; + let builder = configure( + HlukBuilder::from_snapshot(snapshot), + &self.host, + &self.mounts, + policy, + ); + self.sandbox = builder.boot()?; + self.sandbox.drain_output(); + self.socket_hosts = hosts; + self.socket_notice = notice; + Ok(()) + } + + fn run_impl(&mut self, code: &str) -> Result { + { + let mut files = self + .files + .lock() + .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; + // What persists from the last run, and what the host has put + // there since, by the rules a run's output is checked by: the + // guest reaches `/output` itself, through its mount, so it must + // not start on a link or a special file. The rest of + // `prepare_for_run` is resetting `CapFs` handles, of which the + // guest holds none. + files + .validate_output() + .context("refusing to run on what /output holds")?; + self.kept_dirs = + output_dirs(&files).context("failed to list the directories in /output")?; + } + self.sync_socket_policy()?; + self.host.take_stderr()?; + self.host + .blobs + .lock() + .map_err(|_| anyhow::anyhow!("blob store poisoned"))? + .clear(); + + let request = self + .host + .envelope(&serde_json::Value::String(code.to_owned())) + .map_err(|e| anyhow::anyhow!(e))?; + let outcome = self.sandbox.call(RUN_FUNCTION, &request); + // The guest's fd 1 and 2: Python's standard output, and what C + // extensions, child processes and the kernel write, each in the + // order written. Standard error first: draining the interleaved + // output drops it. + let raw_stderr = self.sandbox.drain_stderr(); + let raw = self.sandbox.drain_output(); + let mut stderr = self.host.take_stderr()?; + let mut stdout = String::new(); + // Without a clean split, standard output keeps both, interleaved, + // and standard error still has its own. + push_capped( + &mut stdout, + strip_stderr_lines(&raw, &raw_stderr) + .as_deref() + .unwrap_or(&raw), + ); + push_capped(&mut stderr, &raw_stderr); + + if let Some(notice) = self.socket_notice.take() { + stderr.insert_str(0, ¬ice); + } + + // Nothing below returns early: the output limits are enforced, and + // the output kept, whatever the guest returned. + let mut exit_code = match outcome { + Ok(result) => match result.trim().parse::() { + Ok(status) => exit_status(status), + Err(_) => { + push_capped(&mut stderr, &format!("unexpected run result {result:?}\n")); + 1 + } + }, + Err(error) => { + // The guest itself failed (crashed, deadlocked, exited): + // report it like the other backends report a trap. + push_capped(&mut stderr, &format!("{error}\n")); + -1 + } + }; + if let Some(violation) = self.check_output_limits() { + push_capped(&mut stderr, &format!("{violation}\n")); + if exit_code == 0 { + exit_code = 1; + } + } + Ok(ExecutionResult { + stdout, + stderr, + exit_code, + }) + } + + /// The guest writes `/output` through hostfs, which `CapFs` does not + /// see, so `CapFs` checks it once the run is over, by the rules it + /// applies to writes made through it: the filesystem limits, no + /// symbolic or hard links, no special files, and its traversal bounds + /// (64 levels, 100,000 entries). A run that breaks them fails and + /// `/output` is cleared, earlier runs' files with it: what the guest + /// broke them with cannot be told from the rest. + /// + /// Returns what to say in the run's stderr, if anything. + fn check_output_limits(&self) -> Option { + let Ok(mut files) = self.files.lock() else { + return Some("[hyperlight-sandbox] /output could not be checked".into()); + }; + let error = files.validate_output().err()?; + // Cleared by this backend, which no traversal limit stops, so + // whatever tree the guest made goes; then `CapFs` resynchronises on + // what is left. + let cleared = clear_output(&files, &self.kept_dirs) + .map_err(anyhow::Error::from) + .and_then(|()| files.clear_output_files()); + Some(match cleared { + Ok(()) => format!( + "[hyperlight-sandbox] /output exceeded the filesystem limits ({error:#}); \ + its files were removed" + ), + Err(clear_error) => format!( + "[hyperlight-sandbox] /output exceeded the filesystem limits ({error:#}); \ + removing its files failed: {clear_error:#}" + ), + }) + } +} + +/// Deepest directory nesting `CapFs` walks in `/output`. +const MAX_OUTPUT_DEPTH: usize = 64; + +/// The directories in `/output` now, relative to it, through the handle +/// `CapFs` opened on it. +fn output_dirs(files: &CapFs) -> Result> { + let mut found = HashSet::new(); + let Some(root) = files.dir_by_guest_path("/output") else { + return Ok(found); + }; + // Within `CapFs`'s bounds: `validate_output` just checked the tree. + let mut dirs = vec![(root.cap_std().try_clone()?, PathBuf::new())]; + while let Some((dir, at)) = dirs.pop() { + for entry in dir.entries()? { + let entry = entry?; + if entry.file_type()?.is_dir() { + let path = at.join(entry.file_name()); + if path.components().count() <= MAX_OUTPUT_DEPTH { + dirs.push((dir.open_dir(entry.file_name())?, path.clone())); + found.insert(path); + } + } + } + } + Ok(found) +} + +/// Remove what the guest left in `/output`: every file and link, and +/// every directory but `keep` (those the run started with), however +/// deep. Works through +/// the handle `CapFs` opened, so neither the process's working directory +/// nor a component of the path swapped since can redirect it, and follows +/// no symbolic links. +fn clear_output(files: &CapFs, keep: &HashSet) -> std::io::Result<()> { + match files.dir_by_guest_path("/output") { + Some(root) => clear_dir(root.cap_std(), keep, Path::new("")), + None => Ok(()), + } +} + +fn clear_dir(dir: &CapDir, keep: &HashSet, at: &Path) -> std::io::Result<()> { + for entry in dir.entries()? { + let entry = entry?; + let name = entry.file_name(); + let file_type = entry.file_type()?; + if file_type.is_dir() { + let path = at.join(&name); + if keep.contains(&path) { + clear_dir(&dir.open_dir(&name)?, keep, &path)?; + } else { + dir.remove_dir_all(&name)?; + } + } else if let Err(error) = dir.remove_file(&name) { + // A directory symbolic link or junction (Windows) goes with + // remove_dir; it does not touch the target. + if !file_type.is_symlink() { + return Err(error); + } + dir.remove_dir(&name)?; + } + } + Ok(()) +} + +/// A run's exit status from the bridge, which reports it as a CPython +/// process would exit (0 to 255), so it never meets -1, a guest failure. +/// Anything else is not the bridge's doing and counts as a failure. +fn exit_status(status: i64) -> i32 { + u8::try_from(status).map_or(1, i32::from) +} + +/// Standard output alone, from the interleaved guest output and its +/// standard error. +/// +/// hyperlight-unikraft captures standard error on its own but standard +/// output only interleaved with it. Taking out the standard error lines in +/// order recovers it whenever standard error was written in whole lines, +/// as kernel messages and most programs' errors are: a standard output +/// line identical to a pending standard error line can only trade places +/// with it, as a Python standard output line can with a native standard +/// error line written next to it. Otherwise `None`, and the caller keeps +/// the interleaved text. +fn strip_stderr_lines(output: &str, stderr: &str) -> Option { + if stderr.is_empty() { + return Some(output.to_owned()); + } + let mut pending = stderr.split_inclusive('\n').peekable(); + let mut stdout = String::with_capacity(output.len().saturating_sub(stderr.len())); + for line in output.split_inclusive('\n') { + if pending.peek() == Some(&line) { + pending.next(); + } else { + stdout.push_str(line); + } + } + pending.peek().is_none().then_some(stdout) +} + +/// `/input` read-only, and `/output` as close to its `CapFs` permissions as +/// a hostfs mount, read-only or read-write, can be: read-write when files +/// may be created and written, else read-only when it may be read, else +/// not mounted. Write-only cannot be had: the guest can read back what it +/// wrote. +fn mounts_for(files: &Mutex) -> Result> { + let files = files + .lock() + .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; + // Resolved now, as `CapFs` resolved them: a re-boot remounts them, + // and a relative path would then follow the process's working + // directory elsewhere. + let absolute = |path: &Path| { + std::fs::canonicalize(path).with_context(|| format!("cannot resolve {}", path.display())) + }; + let mut mounts = Vec::new(); + if let Some(input) = files.input_path() { + mounts.push(Mount::ro(absolute(input)?, "/input")); + } + if let (Some(output), Some(dir)) = (files.output_path(), files.dir_by_guest_path("/output")) { + let output = absolute(output)?; + if dir.perms().contains(DirPerms::MUTATE) && dir.file_perms().contains(FilePerms::WRITE) { + mounts.push(Mount::rw(output, "/output")); + } else if dir.perms().contains(DirPerms::READ) { + mounts.push(Mount::ro(output, "/output")); + } + } + Ok(mounts) +} + +/// The host names guest sockets may reach: none unless native networking +/// is on, else those of every `allow_domain` entry. +fn socket_hosts(native: bool, network: &Mutex) -> Result> { + if !native { + return Ok(Vec::new()); + } + let network = network + .lock() + .map_err(|_| anyhow::anyhow!("network mutex poisoned"))?; + let mut hosts: Vec = network + .permissions() + .iter() + .filter_map(|p| p.url.host().map(|host| allowlist_host(&host))) + .collect(); + hosts.sort(); + hosts.dedup(); + Ok(hosts) +} + +/// A URL's host as hyperlight-unikraft's allowlist takes it: a name, or an +/// address without the brackets a URL puts around IPv6. +fn allowlist_host(host: &url::Host<&str>) -> String { + match host { + url::Host::Domain(name) => (*name).to_owned(), + url::Host::Ipv4(address) => address.to_string(), + // As an IPv4 address when it is one: the allowlist compares + // addresses in that form. + url::Host::Ipv6(address) => address + .to_ipv4_mapped() + .map_or_else(|| address.to_string(), |v4| v4.to_string()), + } +} + +/// Refuse to mount a directory other than the one `CapFs` opened for it. +/// +/// hostfs opens each mount by path when the guest boots and keeps that +/// directory until the next boot (a restore keeps it too), while `CapFs` +/// checks and clears the one it opened at build. So this is checked before +/// each boot, the first and the re-boot an allowlist change makes: were the +/// path replaced since the build (`rm -rf out && mkdir out`), the guest +/// would write a directory nothing checks. Between boots, replacing it +/// changes nothing for either side. +fn check_mounts(files: &Mutex, mounts: &[Mount]) -> Result<()> { + let files = files + .lock() + .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; + for mount in mounts { + let same = files + .is_same_dir(&mount.guest_path, &mount.host_path) + .with_context(|| { + format!( + "cannot check that {} ({}) is the directory the sandbox was built with", + mount.guest_path, + mount.host_path.display() + ) + })?; + anyhow::ensure!( + same, + "{} ({}) was replaced since the sandbox was built; restore the original \ + directory or build a new sandbox", + mount.guest_path, + mount.host_path.display() + ); + } + Ok(()) +} + +/// The guest's socket policy for `hosts`, and a note naming those of them +/// that did not resolve, which the policy leaves out. +fn socket_policy(hosts: &[String]) -> (Option, Option) { + if hosts.is_empty() { + return (None, None); + } + let mut reachable: Vec<&String> = hosts + .iter() + .filter(|host| AllowList::from_hosts(&[host.as_str()]).is_ok()) + .collect(); + // Resolved again as one list: drop any that stopped resolving since. + let policy = loop { + if reachable.is_empty() { + break None; + } + match AllowList::from_hosts(&reachable) { + Ok(allow) => break Some(allow), + Err(e) => reachable.retain(|host| host.as_str() != e.host()), + } + }; + let unresolved: Vec<&str> = hosts + .iter() + .filter(|host| !reachable.contains(host)) + .map(String::as_str) + .collect(); + let notice = (!unresolved.is_empty()).then(|| { + format!( + "[hyperlight-sandbox] native network: {} did not resolve; guest sockets cannot \ + reach it until the allowlist changes\n", + unresolved.join(", ") + ) + }); + (policy, notice) +} + +fn configure( + builder: HlukBuilder, + host: &HostState, + mounts: &[Mount], + policy: Option, +) -> HlukBuilder { + // Raw guest output belongs in the run's result, not in the embedding + // process's stdout, and is held to the same cap as the rest of it. + let mut builder = host + .register(builder) + .echo_output(false) + .output_limit(MAX_OUTPUT_BYTES) + .mounts(mounts.iter().cloned()); + if let Some(allow) = policy { + builder = builder.network(NetworkPolicy::AllowList(allow)); + } + builder +} + +impl GuestSandbox for UnikraftGuestSandbox { + type SnapshotData = HlukSnapshot; + + fn run(&mut self, code: &str) -> Result { + self.run_impl(code) + } + + fn snapshot(&mut self) -> Result> { + let snapshot = self + .sandbox + .snapshot() + .map_err(|e| anyhow::anyhow!("snapshot failed: {e}"))?; + Ok(Snapshot::new("hyperlight-unikraft", snapshot)) + } + + fn restore(&mut self, snapshot: &Snapshot) -> Result<()> { + self.sandbox + .restore(snapshot.snapshot().clone()) + .map_err(|e| anyhow::anyhow!("restore failed: {e}"))?; + self.sandbox.drain_output(); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{allowlist_host, exit_status, strip_stderr_lines, uploaded}; + + #[test] + fn exit_statuses_outside_a_byte_are_failures() { + assert_eq!(exit_status(3), 3); + assert_eq!(exit_status(255), 255); + assert_eq!(exit_status(-1), 1); + assert_eq!(exit_status(256), 1); + } + + #[test] + fn only_an_upload_marker_names_an_upload() { + assert_eq!(uploaded(r#"[{"upload": 7}]"#), Some(7)); + assert_eq!(uploaded(r#"["echo", {"upload": 7}]"#), None); + assert_eq!(uploaded(r#"[{"upload": 7, "x": 1}]"#), None); + assert_eq!(uploaded(r#"[{"upload": "7"}]"#), None); + } + + #[test] + fn stderr_lines_are_taken_out_of_the_output() { + let output = "a\n[1] ERR: x\nb\n[2] ERR: y\nc"; + let stderr = "[1] ERR: x\n[2] ERR: y\n"; + assert_eq!( + strip_stderr_lines(output, stderr).as_deref(), + Some("a\nb\nc") + ); + assert_eq!(strip_stderr_lines("a\n", "").as_deref(), Some("a\n")); + } + + #[test] + fn stderr_not_in_whole_lines_leaves_the_output_alone() { + // A partial stderr write joined a stdout line: no clean split. + assert_eq!(strip_stderr_lines("out err\n", "err\n"), None); + } + + #[test] + fn allowlist_hosts_have_no_ipv6_brackets() { + let host = |url: &str| allowlist_host(&url::Url::parse(url).unwrap().host().unwrap()); + assert_eq!( + host("https://[2606:4700:4700::1111]/"), + "2606:4700:4700::1111" + ); + assert_eq!(host("http://[::ffff:1.2.3.4]/"), "1.2.3.4"); + assert_eq!(host("http://127.0.0.1:8080"), "127.0.0.1"); + assert_eq!(host("https://example.com"), "example.com"); + } +} diff --git a/src/unikraft_sandbox/src/prelude.py b/src/unikraft_sandbox/src/prelude.py new file mode 100644 index 0000000..5ebb0f9 --- /dev/null +++ b/src/unikraft_sandbox/src/prelude.py @@ -0,0 +1,295 @@ +# Guest-side bridge for the hyperlight-unikraft backend. +# +# Run once in the guest's __main__ right after boot. It gives guest code the +# same globals the Wasm Python guest has (call_tool, http_get, http_post) and +# defines the guest function the host calls for every Sandbox::run. +# +# Everything the host hands over travels as a hyperlight host function call, +# which carries at most 64 KiB each way. Replies larger than that arrive as a +# blob id the guest reads back in chunks (see `_hls_unwrap`), requests larger +# than that are uploaded in pieces first (see `_hls_call`), and standard +# error is flushed to the host in chunks that fit. Standard output is the +# guest's own (see `_hls_stdout`). + +import base64 as _hls_b64 +import io as _hls_io +import json as _hls_json +import os as _hls_os +import stat as _hls_stat +import sys as _hls_sys +import traceback as _hls_tb +import types as _hls_types + +import hyperlight as _hls_hl + +# Characters of output per host call. hyperlight.call sends its arguments as +# ASCII JSON, which spells a character outside the BMP (an emoji) as two +# \uXXXX escapes, 12 bytes: 4096 characters stay under 48 KiB, well within +# the 64 KiB host call limit. +_HLS_CHUNK_CHARS = 4 * 1024 + +# Largest request, as ASCII JSON, sent in one host call; larger ones are +# uploaded in pieces of half that, since a piece is a JSON string inside the +# call's own JSON, where a quote or backslash doubles. +_HLS_CALL_BYTES = 48 * 1024 + +# What hyperlight.call raises on a thread other than the one serving the +# call. Any other error is a real failure and must surface. +_HLS_WRONG_THREAD = "only from the thread running a call" + + +def _hls_call(function, *args): + """hyperlight.call, for arguments of any size.""" + payload = _hls_json.dumps(list(args)) + if len(payload) <= _HLS_CALL_BYTES: + return _hls_hl.call(function, *args) + upload, step = None, _HLS_CALL_BYTES // 2 + for start in range(0, len(payload), step): + upload = _hls_hl.call("sandbox.upload", upload, payload[start : start + step]) + return _hls_hl.call(function, {"upload": upload}) + + +def _hls_unwrap(reply): + """Decode a host reply envelope: {"inline": v} or {"blob": id, "len": n}.""" + if "inline" in reply: + return reply["inline"] + blob, total = reply["blob"], reply["len"] + parts, offset = [], 0 + while offset < total: + chunk = _hls_b64.b64decode(_hls_hl.call("sandbox.blob_read", blob, offset)) + if not chunk: + raise RuntimeError(f"host blob {blob} ended at {offset} of {total} bytes") + parts.append(chunk) + offset += len(chunk) + return _hls_json.loads(b"".join(parts)) + + +class _HlsHostStream(_hls_io.TextIOBase): + """A text stream whose writes go to a host function. + + Buffered so a print loop costs one VM exit per chunk rather than per + call. Only the thread serving a call may make host calls, so a write + from another thread stays buffered until the serving thread flushes. + """ + + encoding = "utf-8" + errors = "strict" + + def __init__(self, function): + super().__init__() + self._function = function + self._buffer = [] + self._size = 0 + + def writable(self): + return True + + def isatty(self): + return False + + def write(self, text): + if not isinstance(text, str): + raise TypeError(f"write() argument must be str, not {type(text).__name__}") + self._buffer.append(text) + self._size += len(text) + if self._size >= _HLS_CHUNK_CHARS: + self.flush() + return len(text) + + def flush(self): + if not self._buffer: + return + data = "".join(self._buffer) + self._buffer, self._size = [], 0 + for start in range(0, len(data), _HLS_CHUNK_CHARS): + try: + _hls_hl.call(self._function, data[start : start + _HLS_CHUNK_CHARS]) + except RuntimeError as e: + if _HLS_WRONG_THREAD not in str(e): + raise + # Not the serving thread: keep the rest for the next flush. + rest = data[start:] + self._buffer, self._size = [rest], len(rest) + return + + +def call_tool(tool_name, **kwargs): + """Call a host-registered tool by name with keyword arguments.""" + try: + reply = _hls_call("sandbox.call_tool", tool_name, kwargs) + except _hls_hl.HostError as e: + raise RuntimeError(f"Tool '{tool_name}' failed: {e}") from None + return _hls_unwrap(reply) + + +def _hls_http(method, url, body="", content_type=""): + headers = {"user-agent": "hyperlight-sandbox/0.1"} + if content_type: + headers["content-type"] = content_type + try: + reply = _hls_call("sandbox.http", method, url, headers, body or None) + except _hls_hl.HostError as e: + raise OSError(f"HTTP request failed: {e}") from None + return _hls_unwrap(reply) + + +def http_get(url): + """HTTP GET through the host's allowlist. Returns {"status", "headers", "body"}.""" + return _hls_http("GET", url) + + +def http_post(url, body="", content_type="application/json"): + """HTTP POST through the host's allowlist. Returns {"status", "headers", "body"}.""" + return _hls_http("POST", url, body=body, content_type=content_type) + + +# Guest code runs in its own namespace, as in the Wasm guest, so it cannot +# shadow the bridge below; it persists across runs until a restore. A +# module's, registered as `__main__` while guest code runs, so what looks +# `__main__` up by name, as pickle does for a class guest code defined, +# finds guest code's. +_HLS_MAIN = _hls_types.ModuleType("__main__") +_HLS_GLOBALS = _HLS_MAIN.__dict__ +_HLS_GLOBALS.update( + { + "__builtins__": __builtins__, + "call_tool": call_tool, + "http_get": http_get, + "http_post": http_post, + } +) + + +def _hls_exit_status(code, stderr): + """The status a CPython process would exit with for sys.exit(code).""" + if code is None: + return 0 + if isinstance(code, int): # bool included + # exit() keeps the low byte; a code past a C long exits with -1. + return code & 0xFF if -(2**63) <= code < 2**63 else 0xFF + print(code, file=stderr) + return 1 + + +# Standard output goes to the guest's console, where C extensions and child +# processes write theirs to fd 1, so all of it keeps the order it was +# written in, as it does in a CPython process whose standard output is a +# pipe: buffered, flushed at the end of each run, and strict about what it +# can encode. Standard error goes to the host on its own, so the two are +# told apart exactly. Python's standard output is not fd 1 itself, though: +# guest code redirecting or closing fd 1 leaves it alone. +_HLS_CONSOLE = "/dev/stdout" + + +class _HlsConsole(_hls_io.RawIOBase): + """The guest's console, through a descriptor of its own. + + Checked before every write, and opened again when it no longer is the + console: guest code may close it (its number is `sys.stdout.fileno()`), + and a file it opens may then take the number, which must not get the + output. Not inherited by child processes. + """ + + def __init__(self): + super().__init__() + self._fd = None + self._identity = None + + def writable(self): + return True + + @staticmethod + def _identity_of(fd): + # The type too: every filesystem here is device 0, and inode numbers + # are only unique within one, so a file in /output can have the + # console's. + status = _hls_os.fstat(fd) + return status.st_dev, status.st_ino, status.st_mode + + def _console_fd(self): + """Our descriptor, if it still is the console.""" + if self._fd is None: + return None + try: + return self._fd if self._identity_of(self._fd) == self._identity else None + except OSError: + return None + + def fileno(self): + fd = self._console_fd() + if fd is not None: + return fd + fd = _hls_os.open(_HLS_CONSOLE, _hls_os.O_WRONLY | _hls_os.O_CLOEXEC) + identity = self._identity_of(fd) + if not _hls_stat.S_ISCHR(identity[2]): + _hls_os.close(fd) + raise OSError(f"{_HLS_CONSOLE} is not the console") + self._fd, self._identity = fd, identity + return fd + + def write(self, data): + return _hls_os.write(self.fileno(), data) + + def close(self): + # Ours only while it still is the console: a number guest code + # closed may be another file's by now. + fd = self._console_fd() + if fd is not None: + _hls_os.close(fd) + self._fd = None + super().close() + + +_HLS_STDOUT = None + + +def _hls_usable(stream): + try: + return not stream.closed + except ValueError: # detached + return False + + +def _hls_stdout(): + """The stream for a run's standard output: one shared from run to run, + as a process's is, so what holds on to it (a logging handler) keeps + working, but made again if a run closed or detached it.""" + global _HLS_STDOUT + if _HLS_STDOUT is None or not _hls_usable(_HLS_STDOUT): + _HLS_STDOUT = _hls_io.TextIOWrapper(_hls_io.BufferedWriter(_HlsConsole()), encoding="utf-8") + _hls_sys.__stdout__ = _HLS_STDOUT + return _HLS_STDOUT + + +def __hyperlight_sandbox_run(request): + code = _hls_unwrap(request) + stderr = _HlsHostStream("sandbox.stderr") + saved = _hls_sys.stdout, _hls_sys.stderr, _hls_sys.modules.get("__main__") + _hls_sys.stderr = stderr + _hls_sys.modules["__main__"] = _HLS_MAIN + stdout = None + exit_code = 0 + try: + stdout = _hls_sys.stdout = _hls_stdout() + exec(compile(code, "", "exec"), _HLS_GLOBALS) + except SystemExit as e: + exit_code = _hls_exit_status(e.code, stderr) + except BaseException as e: + # Drop this function's own frame: the traceback starts in guest code. + _hls_tb.print_exception(type(e), e, e.__traceback__.tb_next, file=stderr) + exit_code = 1 + finally: + _hls_sys.stdout, _hls_sys.stderr, main = saved + if main is None: + _hls_sys.modules.pop("__main__", None) + else: + _hls_sys.modules["__main__"] = main + try: + # Closing it flushed it, as at a process's exit. + if stdout is not None and _hls_usable(stdout): + stdout.flush() + except Exception as e: + print(f"standard output could not be flushed: {e!r}", file=stderr) + exit_code = exit_code or 1 + stderr.flush() + return exit_code diff --git a/src/unikraft_sandbox/tests/python_image.rs b/src/unikraft_sandbox/tests/python_image.rs new file mode 100644 index 0000000..c458fa2 --- /dev/null +++ b/src/unikraft_sandbox/tests/python_image.rs @@ -0,0 +1,640 @@ +//! End-to-end tests against the published Python rootfs images. +//! +//! They need KVM (or MSHV / WHP) and the images the `hluk` CLI caches +//! (`hluk run --runtime python`, `hluk run --runtime agent`); without a +//! usable hypervisor or an image a test says so and passes, as there is +//! nothing to run it on. + +use hyperlight_sandbox::{Sandbox, SandboxBuilder, ToolRegistry}; +use hyperlight_unikraft_sandbox::{Unikraft, cached_rootfs}; + +fn rootfs(runtime: &str) -> Option { + if !hyperlight_unikraft::hyperlight_host::is_hypervisor_present() { + eprintln!("skipping: no usable hypervisor"); + return None; + } + match cached_rootfs(runtime) { + Ok(path) => Some(path.display().to_string()), + Err(e) => { + eprintln!("skipping: {e}"); + None + } + } +} + +fn python_sandbox( + tools: ToolRegistry, + input: Option<&std::path::Path>, +) -> Option> { + let rootfs = rootfs("python")?; + let mut builder = SandboxBuilder::new() + .module_path(rootfs) + .with_tools(tools) + .temp_output(); + if let Some(input) = input { + builder = builder.input_dir(input); + } + Some(builder.guest(Unikraft::new()).build().expect("boot")) +} + +#[test] +fn run_semantics_match_the_other_backends() { + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("echo", Ok); + // A reply over the 64 KiB host call limit takes the blob path. + tools.register_unvalidated("big", |_| Ok(serde_json::json!("x".repeat(300_000)))); + let input = tempfile::tempdir().unwrap(); + std::fs::write(input.path().join("in.txt"), "from the host").unwrap(); + let Some(mut sandbox) = python_sandbox(tools, Some(input.path())) else { + return; + }; + + // stdout and stderr are separate, and an exception is exit code 1. + let r = sandbox + .run("import sys\nprint('out')\nprint('err', file=sys.stderr)\nraise ValueError('boom')") + .unwrap(); + assert_eq!(r.stdout, "out\n"); + assert!(r.stderr.starts_with("err\n"), "{:?}", r.stderr); + assert!(r.stderr.contains("ValueError: boom"), "{:?}", r.stderr); + assert_eq!(r.exit_code, 1); + + // sys.exit carries its status; sys.exit() is success. + assert_eq!(sandbox.run("import sys; sys.exit(3)").unwrap().exit_code, 3); + assert_eq!(sandbox.run("import sys; sys.exit()").unwrap().exit_code, 0); + + // State persists from one run to the next. + sandbox.run("x = 20").unwrap(); + assert_eq!(sandbox.run("print(x + 1)").unwrap().stdout, "21\n"); + + // Tools, small and large, and their errors. + let r = sandbox + .run( + "print(call_tool('echo', a=1, s='hi'))\n\ + print(len(call_tool('big')))\n\ + try:\n call_tool('missing')\nexcept RuntimeError as e:\n print('error:', e)", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let lines: Vec<&str> = r.stdout.lines().collect(); + assert_eq!(lines[0], "{'a': 1, 's': 'hi'}"); + assert_eq!(lines[1], "300000"); + assert!( + lines[2].starts_with("error: Tool 'missing' failed"), + "{}", + lines[2] + ); + + // Code and output over the host call limit. + let code = format!( + "s = {:?}\nprint(len(s))\nprint('y' * 200000)", + "z".repeat(100_000) + ); + let r = sandbox.run(&code).unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.starts_with("100000\n")); + assert_eq!(r.stdout.len(), "100000\n".len() + 200_001); + + // /input is read-only, /output is writable and kept between runs. + let r = sandbox + .run( + "print(open('/input/in.txt').read())\n\ + open('/output/out.txt', 'w').write('done')\n\ + try:\n open('/input/new.txt', 'w')\nexcept OSError as e:\n print('ro:', e.errno)", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(r.stdout.starts_with("from the host\nro: "), "{}", r.stdout); + assert_eq!( + sandbox.get_output_files().unwrap(), + vec!["out.txt".to_string()] + ); + let r = sandbox + .run("print(open('/output/out.txt').read())") + .unwrap(); + assert_eq!(r.stdout, "done\n", "{}", r.stderr); + + // Network is off unless allowed, and the denial is the host's. + let r = sandbox + .run("try:\n http_get('https://example.com/')\nexcept OSError as e:\n print(e)") + .unwrap(); + assert!(r.stdout.contains("denied"), "{}", r.stdout); + + // Snapshot / restore rewinds the interpreter, and clears /output. + let snapshot = sandbox.snapshot().unwrap(); + sandbox.run("x = 99").unwrap(); + sandbox.restore(&snapshot).unwrap(); + assert_eq!(sandbox.run("print(x)").unwrap().stdout, "20\n"); + assert!(sandbox.get_output_files().unwrap().is_empty()); +} + +#[test] +fn python_and_native_output_keep_their_order() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + let r = sandbox + .run("import os\nprint('before', flush=True)\nos.write(1, b'middle\\n')\nprint('after')") + .unwrap(); + assert_eq!(r.stdout, "before\nmiddle\nafter\n", "{}", r.stderr); + // No clean split when standard error joins a partial standard output + // line: standard output keeps both, and standard error its own. + let r = sandbox + .run("import os\nos.write(1, b'out ')\nos.write(2, b'err\\n')") + .unwrap(); + assert_eq!(r.stdout, "out err\n"); + assert_eq!(r.stderr, "err\n"); + // Closing standard output, or fd 1, costs the run nothing after it. + let r = sandbox + .run("import os, sys\nprint('gone')\nsys.stdout.close()") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let r = sandbox.run("import os\nos.close(1)").unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let r = sandbox + .run("import sys\nprint('back')\nprint('err', file=sys.stderr)") + .unwrap(); + assert_eq!( + (r.exit_code, r.stdout.as_str(), r.stderr.as_str()), + (0, "back\n", "err\n") + ); + // Nor does detaching it, or closing its descriptor; and a file that + // takes the descriptor's number gets none of the output. + let r = sandbox + .run("import io, sys\nsys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding='utf-8')\nprint('detached')") + .unwrap(); + assert_eq!( + (r.exit_code, r.stdout.as_str()), + (0, "detached\n"), + "{}", + r.stderr + ); + let r = sandbox + .run("import os, sys\nos.close(sys.stdout.fileno())\nf = open('/tmp/taken', 'w')") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let r = sandbox + .run("print('still here')\nf.close()\nprint(repr(open('/tmp/taken').read()))") + .unwrap(); + assert_eq!(r.stdout, "still here\n''\n", "{}", r.stderr); + // Nor does a file in /output, whose inode the console's may share. + let r = sandbox + .run("import os, sys\nopen('/output/a', 'w').close()\nos.close(sys.stdout.fileno())\ng = open('/output/b', 'w')") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let r = sandbox + .run("print('kept')\ng.close()\nprint(repr(open('/output/b').read()))") + .unwrap(); + assert_eq!(r.stdout, "kept\n''\n", "{}", r.stderr); + // What holds on to standard output keeps it from run to run. + sandbox + .run("import logging, sys\nlog = logging.getLogger('t')\nlog.addHandler(logging.StreamHandler(sys.stdout))") + .unwrap(); + let r = sandbox.run("log.warning('logged')").unwrap(); + assert_eq!(r.stdout, "logged\n", "{}", r.stderr); +} + +#[test] +fn guest_classes_pickle_as_in_a_python_process() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + sandbox + .run("import pickle\nclass Point:\n def __init__(self, x):\n self.x = x\ndata = pickle.dumps(Point(3))") + .unwrap(); + let r = sandbox + .run("print(pickle.loads(data).x, __name__)\nif __name__ == '__main__':\n print('main')") + .unwrap(); + assert_eq!(r.stdout, "3 __main__\nmain\n", "{}", r.stderr); +} + +#[test] +fn output_limits_are_enforced_after_the_run() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let limits = hyperlight_sandbox::FilesystemLimits::new(1024, 4096, 2).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .filesystem_limits(limits) + .temp_output() + .guest(Unikraft::new()) + .build() + .unwrap(); + let r = sandbox + .run("open('/output/big.bin', 'wb').write(b'0' * 2048)") + .unwrap(); + assert_eq!(r.exit_code, 1); + assert!( + r.stderr.contains("exceeded the filesystem limits"), + "{}", + r.stderr + ); + assert!(sandbox.get_output_files().unwrap().is_empty()); +} + +#[test] +fn agent_image_has_the_data_science_stack() { + let Some(rootfs) = rootfs("agent") else { + return; + }; + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .guest(Unikraft::new()) + .build() + .unwrap(); + let r = sandbox + .run( + "import numpy, pandas, scipy, sklearn\n\ + print(int(numpy.arange(10).sum()), len(pandas.DataFrame({'a': [1, 2]})))", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "45 2\n"); +} + +#[test] +fn agent_image_runs_child_processes_and_threads() { + let Some(rootfs) = rootfs("agent") else { + return; + }; + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .guest(Unikraft::new()) + .build() + .unwrap(); + let r = sandbox + .run( + "import subprocess, threading\n\ + def run(args, **kw):\n return subprocess.run(args, capture_output=True, text=True, **kw).stdout.strip()\n\ + print(run(['echo', 'child']))\n\ + print(run('echo a b c | wc -w', shell=True))\n\ + out = []\n\ + t = threading.Thread(target=lambda: out.append(run(['echo', 'thread'])))\n\ + t.start(); t.join()\n\ + print(out[0])\n\ + try:\n subprocess.run(['sleep', '30'], timeout=0.5)\nexcept subprocess.TimeoutExpired:\n print('killed')", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "child\n3\nthread\nkilled\n"); + // The kernel logs each PATH entry a program is not found in; that is + // standard error, kept out of stdout. + assert!(r.stderr.contains("/usr/bin/sleep"), "{}", r.stderr); +} + +#[test] +fn multibyte_output_and_large_requests_get_through() { + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("echo", Ok); + let Some(mut sandbox) = python_sandbox(tools, None) else { + return; + }; + + // An emoji is 12 bytes of JSON: output chunks must still fit a host call. + let r = sandbox + .run("print('😀' * 9000)\nprint('é' * 30000)\nprint('after')") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + format!("{}\n{}\nafter\n", "😀".repeat(9000), "é".repeat(30000)) + ); + + // Tool arguments and HTTP bodies over the host call limit are uploaded. + let r = sandbox + .run( + "print(len(call_tool('echo', s='😀' * 50000)['s']))\n\ + try:\n http_post('https://example.com/', body='x' * 200000)\n\ + except OSError as e:\n print(e)", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + let lines: Vec<&str> = r.stdout.lines().collect(); + assert_eq!(lines[0], "50000"); + // Refused by the allowlist, so the request reached the host whole. + assert!(lines[1].contains("denied"), "{}", r.stdout); +} + +#[test] +fn unread_replies_are_capped_per_run() { + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("mb", |_| Ok(serde_json::json!("x".repeat(1 << 20)))); + let Some(mut sandbox) = python_sandbox(tools, None) else { + return; + }; + // Calling the host function directly leaves each reply unread. + let r = sandbox + .run( + "import hyperlight\n\ + try:\n for i in range(200):\n hyperlight.call('sandbox.call_tool', 'mb', {})\n\ + except hyperlight.HostError as e:\n print(i, e)", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + // 64 MiB of pending replies at most: 63 of these (each 1 MiB and + // its JSON quotes) fit. + assert!(r.stdout.starts_with("63 "), "{}", r.stdout); + assert!(r.stdout.contains("unread replies"), "{}", r.stdout); + // The next run starts with nothing held. + let r = sandbox.run("print(len(call_tool('mb')))").unwrap(); + assert_eq!(r.stdout, "1048576\n", "{}", r.stderr); +} + +#[test] +fn out_of_range_exit_status_keeps_output_and_limits() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let limits = hyperlight_sandbox::FilesystemLimits::new(1024, 4096, 2).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .filesystem_limits(limits) + .temp_output() + .guest(Unikraft::new()) + .build() + .unwrap(); + let r = sandbox + .run( + "open('/output/big.bin', 'wb').write(b'0' * 2048)\n\ + print('before')\nimport sys\nsys.exit(2 ** 40)", + ) + .unwrap(); + assert_eq!(r.exit_code, 1); + assert_eq!(r.stdout, "before\n"); + assert!( + r.stderr.contains("exceeded the filesystem limits"), + "{}", + r.stderr + ); + assert!(sandbox.get_output_files().unwrap().is_empty()); +} + +#[test] +fn symlinks_in_output_are_removed() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + let r = sandbox + .run( + "import os\n\ + os.symlink('../../../../etc/passwd', '/output/escape')\n\ + open('/output/ok.txt', 'w').write('ok')", + ) + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(r.stderr.contains("symbolic links"), "{}", r.stderr); + let output = sandbox.output_path().unwrap().unwrap(); + assert_eq!(std::fs::read_dir(output).unwrap().count(), 0); +} + +#[test] +fn unresolvable_hosts_do_not_break_native_network() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .guest(Unikraft::new().native_network(true)) + .build() + .unwrap(); + sandbox + .allow_domain( + "https://does-not-resolve.invalid", + hyperlight_sandbox::MethodFilter::Any, + ) + .unwrap(); + let r = sandbox.run("print('first')").unwrap(); + assert_eq!((r.exit_code, r.stdout.as_str()), (0, "first\n")); + assert!( + r.stderr.contains("does-not-resolve.invalid"), + "{}", + r.stderr + ); + // Said once, and later runs are unaffected. + let r = sandbox.run("print('second')").unwrap(); + assert_eq!((r.exit_code, r.stdout.as_str()), (0, "second\n")); + assert!(r.stderr.is_empty(), "{}", r.stderr); +} + +#[test] +fn output_trees_past_the_walk_limits_fail_and_are_removed() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + let output = sandbox.output_path().unwrap().unwrap(); + for code in [ + // Deeper than the 64 levels CapFs walks. + "import os\nos.makedirs('/output/' + '/'.join(['d'] * 70))", + // More entries than the 100,000 it walks, directories counting. + "import os\nfor i in range(100100):\n os.mkdir(f'/output/{i}')", + ] { + let r = sandbox.run(code).unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(r.stderr.contains("its files were removed"), "{}", r.stderr); + assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); + // The next run is unaffected. + let r = sandbox.run("open('/output/ok', 'w').write('ok')").unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(sandbox.get_output_files().unwrap(), vec!["ok".to_string()]); + } +} + +#[test] +fn hard_links_in_output_are_removed() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + let r = sandbox + .run("import os\nopen('/output/a', 'w').write('a')\nos.link('/output/a', '/output/b')") + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(r.stderr.contains("hard-linked"), "{}", r.stderr); + assert!(sandbox.get_output_files().unwrap().is_empty()); +} + +#[test] +fn exit_statuses_follow_cpython() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + for (code, status, stderr) in [ + ("None", 0, ""), + ("True", 1, ""), + ("False", 0, ""), + ("-1", 255, ""), + ("256", 0, ""), + ("2 ** 70", 255, ""), + ("'bye'", 1, "bye\n"), + ] { + let r = sandbox + .run(&format!("import sys\nsys.exit({code})")) + .unwrap(); + assert_eq!((r.exit_code, r.stderr.as_str()), (status, stderr), "{code}"); + } +} + +#[test] +fn host_directories_in_output_are_kept() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let output = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(output.path().join("plots/png")).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .output_dir( + output.path(), + hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, + ) + .guest(Unikraft::new()) + .build() + .unwrap(); + let r = sandbox + .run( + "import os\n\ + open('/output/plots/png/p.png', 'w').write('p')\n\ + os.makedirs('/output/guest/deep')\n\ + open('/output/guest/deep/f', 'w').write('f')", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + // Kept from run to run. + sandbox.run("pass").unwrap(); + assert!(output.path().join("plots/png/p.png").is_file()); + assert!(output.path().join("guest/deep/f").is_file()); + // A directory the host stages after the build. + std::fs::create_dir(output.path().join("staged")).unwrap(); + // A run that fails the check clears it: the directories it started + // with stay, emptied; the ones it made go. + let r = sandbox + .run("import os\nos.makedirs('/output/during/run')\nos.symlink('x', '/output/link')") + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(output.path().join("plots/png").is_dir()); + assert!(!output.path().join("plots/png/p.png").exists()); + assert!(output.path().join("guest/deep").is_dir()); + assert!(!output.path().join("guest/deep/f").exists()); + assert!(output.path().join("staged").is_dir()); + assert!(!output.path().join("during").exists()); + assert!(!output.path().join("link").exists()); +} + +#[test] +fn output_the_host_stages_reaches_the_guest() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let output = tempfile::tempdir().unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .output_dir( + output.path(), + hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, + ) + .guest(Unikraft::new()) + .build() + .unwrap(); + std::fs::write(output.path().join("staged.txt"), "from the host").unwrap(); + let r = sandbox + .run("print(open('/output/staged.txt').read())") + .unwrap(); + assert_eq!(r.stdout, "from the host\n", "{}", r.stderr); + // Restoring a snapshot clears what the host staged too. + let snapshot = sandbox.snapshot().unwrap(); + sandbox.restore(&snapshot).unwrap(); + assert!(!output.path().join("staged.txt").exists()); +} + +#[test] +fn a_run_is_refused_on_what_the_host_put_in_output_past_the_rules() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let output = tempfile::tempdir().unwrap(); + let limits = hyperlight_sandbox::FilesystemLimits::new(1024, 4096, 4).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .filesystem_limits(limits) + .output_dir( + output.path(), + hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, + ) + .guest(Unikraft::new()) + .build() + .unwrap(); + let code = "open('/output/ran', 'w').write('ran')"; + + // Past the limits, as the other backends refuse it. + std::fs::write(output.path().join("big.bin"), vec![0u8; 2048]).unwrap(); + let error = sandbox.run(code).unwrap_err(); + assert!( + format!("{error:#}").contains("refusing to run"), + "{error:#}" + ); + assert!(!output.path().join("ran").exists()); + // Left as the host put it. + assert_eq!( + std::fs::read(output.path().join("big.bin")).unwrap().len(), + 2048 + ); + std::fs::remove_file(output.path().join("big.bin")).unwrap(); + + // A link, which the guest would follow through its mount. + #[cfg(unix)] + { + std::os::unix::fs::symlink("/etc/passwd", output.path().join("link")).unwrap(); + let error = sandbox.run(code).unwrap_err(); + assert!(format!("{error:#}").contains("symbolic links"), "{error:#}"); + assert!(!output.path().join("ran").exists()); + std::fs::remove_file(output.path().join("link")).unwrap(); + } + + // Put right, the next run goes ahead. + assert_eq!(sandbox.run(code).unwrap().exit_code, 0); + assert!(output.path().join("ran").is_file()); +} + +#[test] +fn a_replaced_output_directory_is_refused() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let base = tempfile::tempdir().unwrap(); + let output = base.path().join("out"); + std::fs::create_dir(&output).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .output_dir( + &output, + hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, + ) + .guest(Unikraft::new().native_network(true)) + .build() + .unwrap(); + assert_eq!(sandbox.run("pass").unwrap().exit_code, 0); + + // The host swaps the directory for another of the same name. + let kept = base.path().join("kept"); + std::fs::rename(&output, &kept).unwrap(); + std::fs::create_dir(&output).unwrap(); + sandbox + .allow_domain("http://127.0.0.1", hyperlight_sandbox::MethodFilter::Any) + .unwrap(); + let code = "import os\nos.symlink('x', '/output/link')"; + let error = sandbox.run(code).unwrap_err(); + assert!(format!("{error:#}").contains("was replaced"), "{error:#}"); + // Nothing reached the new directory unchecked. + assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); + + // Putting the original back lets the re-boot, and the run, go ahead. + std::fs::remove_dir(&output).unwrap(); + std::fs::rename(&kept, &output).unwrap(); + let r = sandbox.run(code).unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(r.stderr.contains("symbolic links"), "{}", r.stderr); + assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); +} diff --git a/src/unikraft_sandbox/tests/relative_output.rs b/src/unikraft_sandbox/tests/relative_output.rs new file mode 100644 index 0000000..78498c7 --- /dev/null +++ b/src/unikraft_sandbox/tests/relative_output.rs @@ -0,0 +1,72 @@ +//! A relative output directory stays the one the sandbox was built with +//! when the process changes its working directory. +//! +//! On its own in this binary: the working directory is the process's, and +//! the other tests run on threads of theirs. Needs KVM and the cached +//! `python` image, as `python_image.rs` does; skips without either. + +use hyperlight_sandbox::{DirPerms, FilePerms, SandboxBuilder}; +use hyperlight_unikraft_sandbox::{Unikraft, cached_rootfs}; + +#[test] +fn relative_output_dir_ignores_later_chdir() { + if !hyperlight_unikraft::hyperlight_host::is_hypervisor_present() { + eprintln!("skipping: no usable hypervisor"); + return; + } + let rootfs = match cached_rootfs("python") { + Ok(path) => path, + Err(e) => { + eprintln!("skipping: {e}"); + return; + } + }; + let base = tempfile::tempdir().unwrap(); + let (a, b) = (base.path().join("a"), base.path().join("b")); + std::fs::create_dir_all(a.join("out")).unwrap(); + // Same relative path under another directory, with data to keep. + std::fs::create_dir_all(b.join("out/userdata")).unwrap(); + std::fs::write(b.join("out/userdata/precious.txt"), "keep").unwrap(); + + std::env::set_current_dir(&a).unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs.display().to_string()) + .output_dir( + "out", + DirPerms::READ | DirPerms::MUTATE, + FilePerms::READ | FilePerms::WRITE, + ) + .guest(Unikraft::new().native_network(true)) + .build() + .unwrap(); + std::env::set_current_dir(&b).unwrap(); + + let r = sandbox + .run("open('/output/x.txt', 'w').write('x')") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(a.join("out/x.txt").is_file()); + // A symlink the guest leaves fails the run, and the clear that + // follows acts on a/out alone. + let r = sandbox + .run("import os\nos.symlink('x', '/output/link')") + .unwrap(); + assert_eq!(r.exit_code, 1, "{}", r.stderr); + assert!(!a.join("out/x.txt").exists()); + assert_eq!(std::fs::read_dir(a.join("out")).unwrap().count(), 0); + assert_eq!( + std::fs::read_to_string(b.join("out/userdata/precious.txt")).unwrap(), + "keep" + ); + // A change to the allowlist re-boots the guest, remounting /output: + // still a/out. + sandbox + .allow_domain("http://127.0.0.1", hyperlight_sandbox::MethodFilter::Any) + .unwrap(); + let r = sandbox + .run("open('/output/y.txt', 'w').write('y')") + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(a.join("out/y.txt").is_file()); + assert!(!b.join("out/y.txt").exists()); +} From cafc020e31237ce29fb2e495abccc40328052790 Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:15:07 +0000 Subject: [PATCH 04/10] Add the unikraft backend to the Python SDK Sandbox(backend="unikraft") (also "hyperlight-unikraft" or "hluk") runs code in a hyperlight-unikraft micro-VM through the new hyperlight-sandbox-backend-unikraft package, installed with hyperlight-sandbox[unikraft]. module names a published image the hluk CLI has cached ("agent" by default, or "python"), module_path any rootfs image; not both. scratch_size sets the guest memory, and native_network lets guest sockets reach the allow_domain hosts. These are refused for the other backends, and heap_size and stack_size for this one, before any backend is imported; SandboxEnvironment leaves heap_size and stack_size unset by default to allow it. CodeExecutionTool takes the backend like the others. allow_domain validates its methods when called, before or after the first run, as the other backends do since #249; the network permission tests cover the backend where it is installed, its python image cached and a hypervisor present. The Justfile builds, packages and publishes the backend with the other two, and unikraft-examples runs unikraft_basics.py. Signed-off-by: danbugs --- .github/workflows/publish.yml | 2 +- Cargo.lock | 10 + Cargo.toml | 1 + pyproject.toml | 2 + src/sdk/python/Justfile | 16 +- src/sdk/python/core/README.md | 1 + .../python/core/examples/unikraft_basics.py | 79 ++++++ .../core/hyperlight_sandbox/__init__.py | 77 +++++- src/sdk/python/core/pyproject.toml | 2 +- .../core/tests/test_network_permissions.py | 30 ++- .../core/tests/test_unikraft_options.py | 82 +++++++ src/sdk/python/unikraft_backend/Cargo.toml | 18 ++ src/sdk/python/unikraft_backend/README.md | 3 + .../__init__.py | 10 + .../python/unikraft_backend/pyproject.toml | 30 +++ src/sdk/python/unikraft_backend/src/lib.rs | 227 ++++++++++++++++++ uv.lock | 12 +- 17 files changed, 585 insertions(+), 17 deletions(-) create mode 100644 src/sdk/python/core/examples/unikraft_basics.py create mode 100644 src/sdk/python/core/tests/test_unikraft_options.py create mode 100644 src/sdk/python/unikraft_backend/Cargo.toml create mode 100644 src/sdk/python/unikraft_backend/README.md create mode 100644 src/sdk/python/unikraft_backend/hyperlight_sandbox_backend_unikraft/__init__.py create mode 100644 src/sdk/python/unikraft_backend/pyproject.toml create mode 100644 src/sdk/python/unikraft_backend/src/lib.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 27b77c7..cdb54e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -93,7 +93,7 @@ jobs: - name: Verify manylinux compatibility run: | - for wheel in dist/pythonsdk/{wasm_backend,hyperlight_js_backend}/*.whl; do + for wheel in dist/pythonsdk/{wasm_backend,hyperlight_js_backend,unikraft_backend}/*.whl; do case "$wheel" in *-manylinux_2_28_x86_64.whl) ;; *) diff --git a/Cargo.lock b/Cargo.lock index 4c25188..2df2c70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2087,6 +2087,16 @@ dependencies = [ "pyo3", ] +[[package]] +name = "hyperlight-sandbox-backend-unikraft" +version = "0.8.0" +dependencies = [ + "hyperlight-sandbox", + "hyperlight-sandbox-pyo3-common", + "hyperlight-unikraft-sandbox", + "pyo3", +] + [[package]] name = "hyperlight-sandbox-backend-wasm" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 2a7fa26..05efcaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "src/sdk/python/pyo3_common", "src/sdk/python/wasm_backend", "src/sdk/python/hyperlight_js_backend", + "src/sdk/python/unikraft_backend", "src/sdk/dotnet/ffi", ] resolver = "2" diff --git a/pyproject.toml b/pyproject.toml index 8ceea83..540e2a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ members = [ "src/sdk/python/core", "src/sdk/python/wasm_backend", "src/sdk/python/hyperlight_js_backend", + "src/sdk/python/unikraft_backend", "src/sdk/python/wasm_guests/python_guest", "src/sdk/python/wasm_guests/javascript_guest", ] @@ -20,6 +21,7 @@ members = [ hyperlight-sandbox = { workspace = true } hyperlight-sandbox-backend-wasm = { workspace = true } hyperlight-sandbox-backend-hyperlight-js = { workspace = true } +hyperlight-sandbox-backend-unikraft = { workspace = true } hyperlight-sandbox-python-guest = { workspace = true } hyperlight-sandbox-javascript-guest = { workspace = true } diff --git a/src/sdk/python/Justfile b/src/sdk/python/Justfile index 4150afb..6d50ef9 100644 --- a/src/sdk/python/Justfile +++ b/src/sdk/python/Justfile @@ -23,6 +23,7 @@ dist-root := repo-root + "/dist/pythonsdk" core-dist := dist-root + "/core" wasm-wheels := dist-root + "/wasm_backend" hyperlight-js-wheels := dist-root + "/hyperlight_js_backend" +unikraft-wheels := dist-root + "/unikraft_backend" python-guest-dist := dist-root + "/python_guest" javascript-guest-dist := dist-root + "/javascript_guest" @@ -40,10 +41,10 @@ python-sync-guest-resources: cp {{repo-root}}/src/wasm_sandbox/guests/javascript/js-sandbox.wasm {{repo-root}}/src/sdk/python/wasm_guests/javascript_guest/javascript_guest/resources/js-sandbox.wasm # Install pure Python dependencies from the lockfile into the local venv. -# Skips the two Rust backend packages — those are compiled and installed +# Skips the three Rust backend packages — those are compiled and installed # separately by `python-install-backends` via `maturin develop`. python-sync-env python-version="" $UV_PYTHON=(if python-version != "" { python-version } else { env(uv-python-env, "") }) python-version-arg=(if python-version != "" { python-version-flag + " " + python-version } else { "" }): - uv sync {{python-version-arg}} --frozen --inexact --no-install-package hyperlight-sandbox-backend-wasm --no-install-package hyperlight-sandbox-backend-hyperlight-js + uv sync {{python-version-arg}} --frozen --inexact --no-install-package hyperlight-sandbox-backend-wasm --no-install-package hyperlight-sandbox-backend-hyperlight-js --no-install-package hyperlight-sandbox-backend-unikraft # Install maturin backends into the local venv for development (maturin develop). # This compiles the Rust backend crates and installs them as editable packages @@ -61,7 +62,7 @@ python-clean-backend-artifacts profile="debug": python-develop-backend backend profile="debug" python-version="" $UV_PYTHON=(if python-version != "" { python-version } else { env(uv-python-env, "") }) python-version-arg=(if python-version != "" { python-version-flag + " " + python-version } else { "" }) profile-arg=(if profile == "release" { "--release" } else { "" }): uv run --directory {{python-root}}/{{backend}} --no-sync {{python-version-arg}} maturin develop {{profile-arg}} -python-install-backends profile="debug" python-version="": (python-clean-backend-artifacts profile) (python-develop-backend "wasm_backend" profile python-version) (python-develop-backend "hyperlight_js_backend" profile python-version) +python-install-backends profile="debug" python-version="": (python-clean-backend-artifacts profile) (python-develop-backend "wasm_backend" profile python-version) (python-develop-backend "hyperlight_js_backend" profile python-version) (python-develop-backend "unikraft_backend" profile python-version) python-build profile="debug" python-version="": python-sync-guest-resources (python-sync-env python-version) (python-install-backends profile python-version) @@ -74,6 +75,7 @@ python-dist: python-sync-guest-resources cd {{repo-root}}/src/sdk/python/core && uv run python -m build --outdir {{core-dist}} cd {{repo-root}}/src/sdk/python/wasm_backend && uv run maturin build --release --out {{wasm-wheels}} {{manylinux-arg}} {{interpreters}} cd {{repo-root}}/src/sdk/python/hyperlight_js_backend && uv run maturin build --release --out {{hyperlight-js-wheels}} {{manylinux-arg}} {{interpreters}} + cd {{repo-root}}/src/sdk/python/unikraft_backend && uv run maturin build --release --out {{unikraft-wheels}} {{manylinux-arg}} {{interpreters}} cd {{repo-root}}/src/sdk/python/wasm_guests/python_guest && uv run python -m build --outdir {{python-guest-dist}} cd {{repo-root}}/src/sdk/python/wasm_guests/javascript_guest && uv run python -m build --outdir {{javascript-guest-dist}} @@ -83,12 +85,15 @@ python-dist: python-sync-guest-resources python-dist-backends: python-sync-env -{{rmrf}} {{wasm-wheels}} -{{rmrf}} {{hyperlight-js-wheels}} + -{{rmrf}} {{unikraft-wheels}} cd {{repo-root}}/src/sdk/python/wasm_backend && uv run maturin build --release --out {{wasm-wheels}} {{manylinux-arg}} {{interpreters}} cd {{repo-root}}/src/sdk/python/hyperlight_js_backend && uv run maturin build --release --out {{hyperlight-js-wheels}} {{manylinux-arg}} {{interpreters}} + cd {{repo-root}}/src/sdk/python/unikraft_backend && uv run maturin build --release --out {{unikraft-wheels}} {{manylinux-arg}} {{interpreters}} python-publish repository="pypi": uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{wasm-wheels}}/* uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{hyperlight-js-wheels}}/* + uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{unikraft-wheels}}/* uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{python-guest-dist}}/* uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{javascript-guest-dist}}/* uv publish {{ if repository != "pypi" { "--publish-url https://test.pypi.org/legacy/" } else { "" } }} {{core-dist}}/* @@ -129,6 +134,10 @@ examples: python-build uv run python {{repo-root}}/src/sdk/python/core/examples/jswasm_network_demo.py uv run python {{repo-root}}/src/sdk/python/core/examples/jswasm_filesystem_demo.py +# Needs KVM and the cached `agent` rootfs (`just unikraft pull-images`). +unikraft-examples: python-build + uv run python {{repo-root}}/src/sdk/python/core/examples/unikraft_basics.py + python-sandbox-benchmark: (python-build "release") uv run python {{repo-root}}/src/sdk/python/core/examples/benchmark.py @@ -137,6 +146,7 @@ python-sandbox-benchmark: (python-build "release") clean: -{{rmrf}} {{repo-root}}/src/sdk/python/wasm_backend/target -{{rmrf}} {{repo-root}}/src/sdk/python/hyperlight_js_backend/target + -{{rmrf}} {{repo-root}}/src/sdk/python/unikraft_backend/target #### LINT / FORMAT #### diff --git a/src/sdk/python/core/README.md b/src/sdk/python/core/README.md index 0db0eb0..c8de7a3 100644 --- a/src/sdk/python/core/README.md +++ b/src/sdk/python/core/README.md @@ -34,6 +34,7 @@ Example dependency sets: - `Sandbox(backend="wasm", module="python_guest.path")` requires `hyperlight-sandbox[wasm,python_guest]` - `Sandbox(backend="wasm", module="javascript_guest.path")` requires `hyperlight-sandbox[wasm,javascript_guest]` - `Sandbox(backend="hyperlight-js")` requires `hyperlight-sandbox[hyperlight_js]` +- `Sandbox(backend="unikraft")` requires `hyperlight-sandbox[unikraft]` and a rootfs image the `hluk` CLI has cached (`module="agent"` by default), or `module_path=` to one Use `Sandbox(backend="wasm", module="javascript_guest.path")` to run the packaged JavaScript Wasm guest package. diff --git a/src/sdk/python/core/examples/unikraft_basics.py b/src/sdk/python/core/examples/unikraft_basics.py new file mode 100644 index 0000000..d177eff --- /dev/null +++ b/src/sdk/python/core/examples/unikraft_basics.py @@ -0,0 +1,79 @@ +"""Full CPython with the data-science stack, via the hyperlight-unikraft backend. + +Runs in a Unikraft micro-VM booted from the published ``agent`` rootfs +(CPython 3.12 + numpy, pandas, scipy, scikit-learn). Needs KVM, MSHV or +Hyper-V, and the image in the hluk CLI's cache (``just unikraft pull-images``), +or pass its path with ``module_path=``. +""" + +import sys +import time + +from hyperlight_sandbox import Sandbox + +SALES = [ + {"region": "north", "units": 120, "price": 9.5}, + {"region": "south", "units": 80, "price": 11.0}, + {"region": "north", "units": 45, "price": 9.5}, + {"region": "east", "units": 200, "price": 7.25}, +] + + +def timed(label, fn): + started = time.perf_counter() + value = fn() + print(f"[{label}: {(time.perf_counter() - started) * 1000:.1f} ms]") + return value + + +def main() -> None: + kwargs = {"module_path": sys.argv[1]} if len(sys.argv) > 1 else {"module": "agent"} + sandbox = Sandbox(backend="unikraft", temp_output=True, **kwargs) + sandbox.register_tool("fetch_data", lambda table="": SALES if table == "sales" else []) + sandbox.register_tool("multiply", lambda a=0.0, b=0.0: a * b) + + # The micro-VM boots on the first run. + result = timed( + "boot + first run", + lambda: sandbox.run( + "import sys, numpy as np\n" + "print(f'Python {sys.version.split()[0]}, numpy {np.__version__}')\n" + "print('det =', round(float(np.linalg.det(np.arange(1, 10).reshape(3, 3) + np.eye(3))), 3))" + ), + ) + print(result.stdout, end="") + assert result.success, result.stderr + + sandbox.run("counter = 41") + result = timed("run", lambda: sandbox.run("counter += 1\nprint('counter =', counter)")) + print(result.stdout, end="") + assert "counter = 42" in result.stdout + + result = timed( + "tools + pandas", + lambda: sandbox.run( + "import pandas as pd\n" + "df = pd.DataFrame(call_tool('fetch_data', table='sales'))\n" + "df['revenue'] = df['units'] * df['price']\n" + "print(df.groupby('region')['revenue'].sum().to_string())\n" + "print('6 x 7 =', call_tool('multiply', a=6, b=7))\n" + "df.to_csv('/output/sales.csv')" + ), + ) + print(result.stdout, end="") + assert result.success, result.stderr + print("output files:", sandbox.get_output_files()) + + snapshot = timed("snapshot", sandbox.snapshot) + sandbox.run("counter = 100") + timed("restore", lambda: sandbox.restore(snapshot)) + result = sandbox.run("print('counter =', counter)") + print(result.stdout, end="") + assert "counter = 42" in result.stdout + + result = sandbox.run("1 / 0") + print("exit code", result.exit_code, "|", result.stderr.strip().splitlines()[-1]) + + +if __name__ == "__main__": + main() diff --git a/src/sdk/python/core/hyperlight_sandbox/__init__.py b/src/sdk/python/core/hyperlight_sandbox/__init__.py index 2287515..5bd8dcf 100644 --- a/src/sdk/python/core/hyperlight_sandbox/__init__.py +++ b/src/sdk/python/core/hyperlight_sandbox/__init__.py @@ -34,7 +34,9 @@ def _normalize_backend(backend: str) -> str: return "wasm" if normalized in {"javascript", "js", "hyperlight-js"}: return "hyperlight-js" - raise ValueError(f"Unknown backend '{backend}'. Expected 'wasm' or 'hyperlight-js'.") + if normalized in {"unikraft", "hyperlight-unikraft", "hluk"}: + return "unikraft" + raise ValueError(f"Unknown backend '{backend}'. Expected 'wasm', 'hyperlight-js' or 'unikraft'.") def _load_backend(backend: str): @@ -49,6 +51,16 @@ def _load_backend(backend: str): ) from exc return normalized, NativeWasmSandbox + if normalized == "unikraft": + try: + from hyperlight_sandbox_backend_unikraft import UnikraftSandbox as NativeUnikraftSandbox + except ImportError as exc: + raise ImportError( + "The hyperlight-unikraft backend is not installed. Install hyperlight-sandbox[unikraft] " + "or install the local hyperlight-sandbox-backend-unikraft package." + ) from exc + return normalized, NativeUnikraftSandbox + try: from hyperlight_sandbox_backend_hyperlight_js import JSSandbox as NativeJSSandbox except ImportError as exc: @@ -96,12 +108,15 @@ class SandboxEnvironment: backend: str = "wasm" module: str | None = DEFAULT_MODULE_REF module_path: str | None = None - heap_size: str = field(default_factory=lambda: _DEFAULT_HEAP_SIZE) - stack_size: str = field(default_factory=lambda: _DEFAULT_STACK_SIZE) + # None: the backend's default (the unikraft backend takes none of them). + heap_size: str | None = None + stack_size: str | None = None filesystem_limits: Literal["unlimited"] | None = None max_file_size: str | None = None max_total_size: str | None = None max_file_count: int | None = None + scratch_size: str | None = None + native_network: bool = False class Sandbox: @@ -111,6 +126,28 @@ class Sandbox: ``"64Mi"``. Partial numeric overrides retain balanced defaults for omitted values. Use ``filesystem_limits="unlimited"`` to explicitly disable all filesystem quotas; it cannot be combined with numeric overrides. + + ``backend="unikraft"`` runs full CPython in a hyperlight-unikraft micro-VM. + ``module_path`` is then a rootfs image (initrd CPIO), and ``module`` names a + published image the ``hluk`` CLI has cached (``"agent"`` by default: CPython + with numpy, pandas, scipy and scikit-learn; or ``"python"``). ``heap_size`` + and ``stack_size`` are refused for it: ``scratch_size`` (e.g. ``"2Gi"``) + overrides the guest memory the image was tested with. ``native_network`` + also lets guest sockets (``requests``, ``urllib``) reach the domains passed + to ``allow_domain``, enforced by host name only; a name that does not + resolve is left out, and the next run's stderr says so. ``/input`` and + ``/output`` are host directories mounted at boot. As with the other + backends, files in ``/output`` persist from run to run and ``restore()`` + clears them; open guest files are interpreter state, and stay open too. + A run that leaves a symbolic or hard link, a special file or files past + the filesystem limits in ``/output`` fails, and ``/output`` is cleared of + every file and link (earlier runs' included) and of the directories made + during the run; those it had when the run started are kept. A run is + refused if ``/output`` holds such things when it would start. Exit + statuses are a CPython process's, 0 to 255 (``sys.exit(-1)`` is 255), so + -1 means the guest itself failed. + Tool arguments and results and HTTP bodies of any size are exchanged in + pieces, up to 64 MiB held unread on the host at a time. """ def __init__( @@ -128,6 +165,8 @@ def __init__( max_file_size: str | None = None, max_total_size: str | None = None, max_file_count: int | None = None, + scratch_size: str | None = None, + native_network: bool = False, ) -> None: numeric_limits = (max_file_size, max_total_size, max_file_count) if filesystem_limits is not None and filesystem_limits != "unlimited": @@ -148,6 +187,15 @@ def __init__( if max_file_count < 0: raise ValueError("max_file_count must be non-negative") + # Checked before the backend is imported, so a wrong option is a + # ValueError whichever backends are installed. + unikraft = _normalize_backend(backend) == "unikraft" + if unikraft and (heap_size is not None or stack_size is not None): + raise ValueError("heap_size and stack_size do not apply to the unikraft backend; use scratch_size") + if not unikraft and (scratch_size is not None or native_network): + raise ValueError("scratch_size and native_network apply only to the unikraft backend") + if unikraft and module_path is not None and module not in (None, DEFAULT_MODULE_REF): + raise ValueError("pass either module or module_path, not both") if heap_size is None: heap_size = _DEFAULT_HEAP_SIZE if stack_size is None: @@ -157,10 +205,15 @@ def __init__( if module_path is not None and module == DEFAULT_MODULE_REF: effective_module = None - kwargs: dict[str, Any] = { - "heap_size": heap_size, - "stack_size": stack_size, - } + kwargs: dict[str, Any] = {} + if unikraft: + if scratch_size is not None: + kwargs["scratch_size"] = scratch_size + if native_network: + kwargs["native_network"] = True + else: + kwargs["heap_size"] = heap_size + kwargs["stack_size"] = stack_size if input_dir is not None: kwargs["input_dir"] = input_dir if output_dir is not None: @@ -182,6 +235,14 @@ def __init__( module_path=resolved_module_path, **kwargs, ) + elif normalized_backend == "unikraft": + # The Wasm default module names a Wasm guest package; for this + # backend a module is a published rootfs image name. + if module_path is not None: + kwargs["module_path"] = module_path + elif effective_module is not None and effective_module != DEFAULT_MODULE_REF: + kwargs["runtime"] = effective_module + self._inner = native_cls(**kwargs) else: self._inner = native_cls(**kwargs) @@ -268,6 +329,8 @@ def _get_sandbox(self) -> Sandbox: max_file_size=self.environment.max_file_size, max_total_size=self.environment.max_total_size, max_file_count=self.environment.max_file_count, + scratch_size=self.environment.scratch_size, + native_network=self.environment.native_network, ) for tool_fn in self.tools: self._sandbox.register_tool(tool_fn) diff --git a/src/sdk/python/core/pyproject.toml b/src/sdk/python/core/pyproject.toml index 4525bd1..88a5936 100644 --- a/src/sdk/python/core/pyproject.toml +++ b/src/sdk/python/core/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" license = "Apache-2.0" requires-python = ">=3.10" dependencies = [] -optional-dependencies = { wasm = ["hyperlight-sandbox-backend-wasm>=0.8.0"], hyperlight_js = ["hyperlight-sandbox-backend-hyperlight-js>=0.8.0"], python_guest = ["hyperlight-sandbox-python-guest>=0.8.0"], javascript_guest = ["hyperlight-sandbox-javascript-guest>=0.8.0"], dev = ["atheris>=2.3.0; python_version < '3.13'"] } +optional-dependencies = { wasm = ["hyperlight-sandbox-backend-wasm>=0.8.0"], hyperlight_js = ["hyperlight-sandbox-backend-hyperlight-js>=0.8.0"], unikraft = ["hyperlight-sandbox-backend-unikraft>=0.8.0"], python_guest = ["hyperlight-sandbox-python-guest>=0.8.0"], javascript_guest = ["hyperlight-sandbox-javascript-guest>=0.8.0"], dev = ["atheris>=2.3.0; python_version < '3.13'"] } classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", diff --git a/src/sdk/python/core/tests/test_network_permissions.py b/src/sdk/python/core/tests/test_network_permissions.py index 38f8105..21276b5 100644 --- a/src/sdk/python/core/tests/test_network_permissions.py +++ b/src/sdk/python/core/tests/test_network_permissions.py @@ -1,6 +1,8 @@ """Regression tests for Python network permission validation.""" import multiprocessing +import os +import sys import unittest from http.server import BaseHTTPRequestHandler, HTTPServer @@ -29,6 +31,19 @@ def _serve(connection): server.handle_request() +def _unikraft_available(): + """Whether the unikraft backend can run here: installed, its python image + in the hluk cache, and a hypervisor. Like its Rust end-to-end tests, the + cases skip themselves otherwise.""" + try: + Sandbox(backend="unikraft", module="python") + except (ImportError, RuntimeError): + return False + if sys.platform == "linux": + return os.path.exists("/dev/kvm") or os.path.exists("/dev/mshv") + return sys.platform == "win32" + + class TestPendingNetworkPermissions(unittest.TestCase): @classmethod def setUpClass(cls): @@ -46,7 +61,7 @@ def tearDownClass(cls): cls.server_process.close() def backend_cases(self): - return { + cases = { "wasm": { "get": f"print(http_get('{self.url}')['status'])", "plain": "print('plain')", @@ -56,11 +71,18 @@ def backend_cases(self): "plain": "console.log('plain');", }, } + if _unikraft_available(): + cases["unikraft"] = { + "get": f"print(http_get('{self.url}')['status'])", + "plain": "print('plain')", + "options": {"module": "python"}, + } + return cases def test_valid_queued_rule_works_on_first_run(self): for backend, code in self.backend_cases().items(): with self.subTest(backend=backend): - sandbox = Sandbox(backend=backend) + sandbox = Sandbox(backend=backend, **code.get("options", {})) sandbox.allow_domain(self.url, methods=["GET"]) result = sandbox.run(code["get"]) @@ -70,7 +92,7 @@ def test_valid_queued_rule_works_on_first_run(self): def test_invalid_queued_method_does_not_discard_valid_rule(self): for backend, code in self.backend_cases().items(): with self.subTest(backend=backend): - sandbox = Sandbox(backend=backend) + sandbox = Sandbox(backend=backend, **code.get("options", {})) sandbox.allow_domain(self.url, methods=["GET"]) with self.assertRaisesRegex(RuntimeError, "invalid HTTP method: PROPFIND"): @@ -87,7 +109,7 @@ def test_invalid_queued_method_does_not_discard_valid_rule(self): def test_invalid_method_is_rejected_after_initialization(self): for backend, code in self.backend_cases().items(): with self.subTest(backend=backend): - sandbox = Sandbox(backend=backend) + sandbox = Sandbox(backend=backend, **code.get("options", {})) result = sandbox.run(code["plain"]) self.assertEqual(result.exit_code, 0, result.stderr) diff --git a/src/sdk/python/core/tests/test_unikraft_options.py b/src/sdk/python/core/tests/test_unikraft_options.py new file mode 100644 index 0000000..316bd54 --- /dev/null +++ b/src/sdk/python/core/tests/test_unikraft_options.py @@ -0,0 +1,82 @@ +"""Tests for how Sandbox maps its options onto the unikraft backend.""" + +import unittest +from unittest.mock import patch + +from hyperlight_sandbox import CodeExecutionTool, Sandbox, SandboxEnvironment, _normalize_backend + + +class _FakeNativeSandbox: + calls: list[dict] = [] + + def __init__(self, **kwargs): + self.calls.append(kwargs) + + +class TestUnikraftOptions(unittest.TestCase): + def setUp(self): + _FakeNativeSandbox.calls.clear() + + def _build(self, **kwargs): + with patch( + "hyperlight_sandbox._load_backend", + return_value=("unikraft", _FakeNativeSandbox), + ): + Sandbox(backend="unikraft", **kwargs) + return _FakeNativeSandbox.calls[-1] + + def test_defaults_leave_the_image_to_the_backend(self): + kwargs = self._build() + # The Wasm guest default and Wasm memory sizes do not apply. + self.assertNotIn("runtime", kwargs) + self.assertNotIn("module_path", kwargs) + self.assertNotIn("heap_size", kwargs) + self.assertNotIn("stack_size", kwargs) + + def test_unikraft_options_are_forwarded(self): + kwargs = self._build(module="python", scratch_size="2Gi", native_network=True) + self.assertEqual(kwargs["runtime"], "python") + self.assertEqual(kwargs["scratch_size"], "2Gi") + self.assertTrue(kwargs["native_network"]) + + def test_module_path_names_the_rootfs(self): + kwargs = self._build(module_path="/images/agent.cpio") + self.assertEqual(kwargs["module_path"], "/images/agent.cpio") + self.assertNotIn("runtime", kwargs) + + def test_options_are_checked_before_any_backend_loads(self): + # No backend need be installed to be told an option is wrong. + with patch("hyperlight_sandbox._load_backend", side_effect=ImportError) as load: + with self.assertRaises(ValueError): + Sandbox(backend="hyperlight-js", scratch_size="2Gi") + with self.assertRaises(ValueError): + Sandbox(backend="wasm", native_network=True) + with self.assertRaises(ValueError): + Sandbox(backend="unikraft", heap_size="64Mi") + with self.assertRaises(ValueError): + Sandbox(backend="unikraft", stack_size="1Mi") + with self.assertRaises(ValueError): + Sandbox(backend="unikraft", module="python", module_path="/images/agent.cpio") + load.assert_not_called() + + def test_code_execution_tool_builds_a_unikraft_sandbox(self): + tool = CodeExecutionTool( + environment=SandboxEnvironment(backend="unikraft", module="python", scratch_size="512Mi") + ) + with patch( + "hyperlight_sandbox._load_backend", + return_value=("unikraft", _FakeNativeSandbox), + ): + tool._get_sandbox() + kwargs = _FakeNativeSandbox.calls[-1] + self.assertEqual(kwargs["runtime"], "python") + self.assertEqual(kwargs["scratch_size"], "512Mi") + self.assertNotIn("heap_size", kwargs) + + def test_backend_aliases(self): + for alias in ("unikraft", "hyperlight-unikraft", "hluk"): + self.assertEqual(_normalize_backend(alias), "unikraft") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/sdk/python/unikraft_backend/Cargo.toml b/src/sdk/python/unikraft_backend/Cargo.toml new file mode 100644 index 0000000..906c4dd --- /dev/null +++ b/src/sdk/python/unikraft_backend/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "hyperlight-sandbox-backend-unikraft" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description = "hyperlight-unikraft backend bindings for hyperlight-sandbox" +publish = false + +[lib] +name = "_native_unikraft" +crate-type = ["cdylib"] + +[dependencies] +pyo3.workspace = true +hyperlight-sandbox.workspace = true +hyperlight-unikraft-sandbox.workspace = true +hyperlight-sandbox-pyo3-common.workspace = true diff --git a/src/sdk/python/unikraft_backend/README.md b/src/sdk/python/unikraft_backend/README.md new file mode 100644 index 0000000..67836b3 --- /dev/null +++ b/src/sdk/python/unikraft_backend/README.md @@ -0,0 +1,3 @@ +# hyperlight-sandbox-backend-unikraft + +Native [hyperlight-unikraft](https://github.com/hyperlight-dev/hyperlight-unikraft) backend package for the `hyperlight-sandbox` Python API: full CPython 3.12 (with numpy, pandas, scipy and scikit-learn in the `agent` image) in a Unikraft micro-VM. diff --git a/src/sdk/python/unikraft_backend/hyperlight_sandbox_backend_unikraft/__init__.py b/src/sdk/python/unikraft_backend/hyperlight_sandbox_backend_unikraft/__init__.py new file mode 100644 index 0000000..82d2ffc --- /dev/null +++ b/src/sdk/python/unikraft_backend/hyperlight_sandbox_backend_unikraft/__init__.py @@ -0,0 +1,10 @@ +"""hyperlight-unikraft backend implementation package for hyperlight_sandbox.""" + +from hyperlight_sandbox_backend_unikraft._native_unikraft import ( + PyExecutionResult, + PySnapshot, + UnikraftSandbox, + __version__, +) + +__all__ = ["PyExecutionResult", "PySnapshot", "UnikraftSandbox", "__version__"] diff --git a/src/sdk/python/unikraft_backend/pyproject.toml b/src/sdk/python/unikraft_backend/pyproject.toml new file mode 100644 index 0000000..d64ef2f --- /dev/null +++ b/src/sdk/python/unikraft_backend/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["maturin>=1.13.3,<2.0"] +build-backend = "maturin" + +[project] +name = "hyperlight-sandbox-backend-unikraft" +version = "0.8.0" +description = "hyperlight-unikraft (CPython micro-VM) backend implementation for hyperlight-sandbox" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", + "Programming Language :: Rust", + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", +] + +[tool.maturin] +python-source = "." +features = ["pyo3/extension-module"] +module-name = "hyperlight_sandbox_backend_unikraft._native_unikraft" \ No newline at end of file diff --git a/src/sdk/python/unikraft_backend/src/lib.rs b/src/sdk/python/unikraft_backend/src/lib.rs new file mode 100644 index 0000000..c989795 --- /dev/null +++ b/src/sdk/python/unikraft_backend/src/lib.rs @@ -0,0 +1,227 @@ +use std::collections::HashMap; + +use hyperlight_sandbox::{ + DirPerms, FilePerms, FilesystemLimits, HttpMethod, MethodFilter, Sandbox, SandboxBuilder, +}; +use hyperlight_sandbox_pyo3_common::{ + PyExecutionResult, build_tool_registry, parse_filesystem_limits, parse_size, + parse_tool_registration, +}; +use hyperlight_unikraft_sandbox::{Unikraft, cached_rootfs}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::prelude::*; + +type UnikraftSandboxInner = Sandbox; +type UnikraftSnapshotInner = hyperlight_sandbox::Snapshot<< + ::Sandbox as hyperlight_sandbox::GuestSandbox +>::SnapshotData>; + +#[pyclass] +pub struct PySnapshot { + inner: UnikraftSnapshotInner, +} + +#[pyclass(unsendable)] +pub struct UnikraftSandbox { + inner: Option, + tools: HashMap>, + pending_networks: Vec<(String, MethodFilter)>, + rootfs: String, + guest: Unikraft, + input_dir: Option, + output_dir: Option, + temp_output: bool, + filesystem_limits: Option, +} + +#[pymethods] +impl UnikraftSandbox { + /// `module_path` is a rootfs image (initrd CPIO); without one, the + /// published image named by `runtime` (default `agent`) is taken from + /// the `hluk` CLI's cache. `scratch_size` overrides the guest memory + /// the image was tested with (e.g. `"2Gi"`). `native_network` also lets + /// guest sockets reach the allowed domains (by host name only). + #[new] + #[pyo3(signature = (input_dir=None, output_dir=None, temp_output=false, module_path=None, runtime=None, scratch_size=None, native_network=false, filesystem_limits=None, max_file_size=None, max_total_size=None, max_file_count=None))] + fn new( + input_dir: Option<&str>, + output_dir: Option<&str>, + temp_output: bool, + module_path: Option<&str>, + runtime: Option<&str>, + scratch_size: Option<&str>, + native_network: bool, + filesystem_limits: Option<&str>, + max_file_size: Option<&str>, + max_total_size: Option<&str>, + max_file_count: Option, + ) -> PyResult { + let rootfs = match (module_path, runtime) { + (Some(_), Some(_)) => { + return Err(PyValueError::new_err( + "pass either module_path or runtime, not both", + )); + } + (Some(path), None) => path.to_string(), + (None, runtime) => cached_rootfs(runtime.unwrap_or("agent")) + .map_err(|e| PyRuntimeError::new_err(format!("{e:#}")))? + .display() + .to_string(), + }; + let mut guest = Unikraft::new().native_network(native_network); + if let Some(size) = scratch_size { + let mb = usize::try_from(parse_size(size)? / (1024 * 1024)) + .map_err(|_| PyValueError::new_err("scratch_size is too large"))?; + if mb == 0 { + return Err(PyValueError::new_err("scratch_size must be at least 1Mi")); + } + guest = guest.scratch_mb(mb); + } + Ok(UnikraftSandbox { + inner: None, + tools: HashMap::new(), + pending_networks: Vec::new(), + rootfs, + guest, + input_dir: input_dir.map(|s| s.to_string()), + output_dir: output_dir.map(|s| s.to_string()), + temp_output, + filesystem_limits: parse_filesystem_limits( + filesystem_limits, + max_file_size, + max_total_size, + max_file_count, + )?, + }) + } + + #[pyo3(signature = (name_or_tool, callback=None))] + fn register_tool( + &mut self, + py: Python<'_>, + name_or_tool: Py, + callback: Option>, + ) -> PyResult<()> { + if self.inner.is_some() { + return Err(PyRuntimeError::new_err( + "Cannot register tools after sandbox has been initialized. \ + Register all tools before the first run() call.", + )); + } + let (name, cb) = parse_tool_registration(py, name_or_tool, callback)?; + self.tools.insert(name, cb); + Ok(()) + } + + #[pyo3(signature = (code))] + fn run(&mut self, py: Python<'_>, code: &str) -> PyResult { + if self.inner.is_none() { + let registry = build_tool_registry(py, &mut self.tools)?; + let mut builder = SandboxBuilder::new() + .module_path(&self.rootfs) + .with_tools(registry) + .guest(self.guest); + if let Some(limits) = self.filesystem_limits { + builder = builder.filesystem_limits(limits); + } + if let Some(ref dir) = self.input_dir { + builder = builder.input_dir(dir); + } + if let Some(ref dir) = self.output_dir { + builder = builder.output_dir( + dir, + DirPerms::READ | DirPerms::MUTATE, + FilePerms::READ | FilePerms::WRITE, + ); + } else if self.temp_output { + builder = builder.temp_output(); + } + // Booting the micro-VM takes no Python state: let other + // Python threads run meanwhile. + let mut sandbox = py + .detach(|| builder.build()) + .map_err(|e| PyRuntimeError::new_err(format!("Failed to create sandbox: {e:#}")))?; + for (target, methods) in std::mem::take(&mut self.pending_networks) { + sandbox + .allow_domain(&target, methods) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + } + self.inner = Some(sandbox); + } + let sandbox = self.inner.as_mut().unwrap(); + let result = sandbox + .run(code) + .map_err(|e| PyRuntimeError::new_err(format!("Execution failed: {e}")))?; + Ok(PyExecutionResult { + stdout: result.stdout, + stderr: result.stderr, + exit_code: result.exit_code, + }) + } + + #[pyo3(signature = (target, methods=None))] + fn allow_domain(&mut self, target: &str, methods: Option>) -> PyResult<()> { + let methods = + HttpMethod::parse_list(methods).map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + if let Some(sandbox) = self.inner.as_mut() { + sandbox + .allow_domain(target, methods) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + } else { + self.pending_networks.push((target.to_string(), methods)); + } + Ok(()) + } + + fn snapshot(&mut self) -> PyResult { + let sandbox = self + .inner + .as_mut() + .ok_or_else(|| PyRuntimeError::new_err("Sandbox not initialized"))?; + let snap = sandbox + .snapshot() + .map_err(|e| PyRuntimeError::new_err(format!("Snapshot failed: {e}")))?; + Ok(PySnapshot { inner: snap }) + } + + fn restore(&mut self, snapshot: &PySnapshot) -> PyResult<()> { + let sandbox = self + .inner + .as_mut() + .ok_or_else(|| PyRuntimeError::new_err("Sandbox not initialized"))?; + sandbox + .restore(&snapshot.inner) + .map_err(|e| PyRuntimeError::new_err(format!("Restore failed: {e}")))?; + Ok(()) + } + + fn get_output_files(&self) -> PyResult> { + let sandbox = self + .inner + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("Sandbox not initialized"))?; + sandbox + .get_output_files() + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get output files: {e}"))) + } + + fn output_path(&self) -> PyResult> { + let sandbox = self + .inner + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("Sandbox not initialized"))?; + let path = sandbox + .output_path() + .map_err(|e| PyRuntimeError::new_err(format!("Failed to get output path: {e}")))?; + Ok(path.map(|p| p.display().to_string())) + } +} + +#[pymodule] +fn _native_unikraft(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + Ok(()) +} diff --git a/uv.lock b/uv.lock index 3c0ecf5..092efb9 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,7 @@ prerelease-mode = "allow" members = [ "hyperlight-sandbox", "hyperlight-sandbox-backend-hyperlight-js", + "hyperlight-sandbox-backend-unikraft", "hyperlight-sandbox-backend-wasm", "hyperlight-sandbox-dev", "hyperlight-sandbox-javascript-guest", @@ -344,6 +345,9 @@ javascript-guest = [ python-guest = [ { name = "hyperlight-sandbox-python-guest" }, ] +unikraft = [ + { name = "hyperlight-sandbox-backend-unikraft" }, +] wasm = [ { name = "hyperlight-sandbox-backend-wasm" }, ] @@ -352,17 +356,23 @@ wasm = [ requires-dist = [ { name = "atheris", marker = "python_full_version < '3.13' and extra == 'dev'", specifier = ">=2.3.0" }, { name = "hyperlight-sandbox-backend-hyperlight-js", marker = "extra == 'hyperlight-js'", editable = "src/sdk/python/hyperlight_js_backend" }, + { name = "hyperlight-sandbox-backend-unikraft", marker = "extra == 'unikraft'", editable = "src/sdk/python/unikraft_backend" }, { name = "hyperlight-sandbox-backend-wasm", marker = "extra == 'wasm'", editable = "src/sdk/python/wasm_backend" }, { name = "hyperlight-sandbox-javascript-guest", marker = "extra == 'javascript-guest'", editable = "src/sdk/python/wasm_guests/javascript_guest" }, { name = "hyperlight-sandbox-python-guest", marker = "extra == 'python-guest'", editable = "src/sdk/python/wasm_guests/python_guest" }, ] -provides-extras = ["wasm", "hyperlight-js", "python-guest", "javascript-guest", "dev"] +provides-extras = ["wasm", "hyperlight-js", "unikraft", "python-guest", "javascript-guest", "dev"] [[package]] name = "hyperlight-sandbox-backend-hyperlight-js" version = "0.8.0" source = { editable = "src/sdk/python/hyperlight_js_backend" } +[[package]] +name = "hyperlight-sandbox-backend-unikraft" +version = "0.8.0" +source = { editable = "src/sdk/python/unikraft_backend" } + [[package]] name = "hyperlight-sandbox-backend-wasm" version = "0.8.0" From 59f5f0136a6f4b5fea8ea51812dc50d0413ffb3d Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:15:12 +0000 Subject: [PATCH 05/10] Add the unikraft backend to the .NET SDK SandboxBackend.Unikraft (FFI backend 2) builds a hyperlight-unikraft sandbox from the rootfs image given as the module path. WithScratchSize (hyperlight_sandbox_set_scratch_size) sets its guest memory, at least 1 MiB, in place of the heap and stack sizes, which are refused for it as in the Python SDK; unset, the image's tested size applies. WithNativeNetwork (hyperlight_sandbox_set_native_network) turns on guest sockets, as native_network does in Python. CodeExecutionTool initialises it as it does the Wasm Python guest. Its sandbox is boxed in the FFI's backend enum, being several times the size of the others. Signed-off-by: danbugs --- Cargo.lock | 1 + src/sdk/dotnet/README.md | 29 ++- src/sdk/dotnet/core/Api/Sandbox.cs | 16 +- src/sdk/dotnet/core/Api/SandboxBackend.cs | 22 ++ src/sdk/dotnet/core/Api/SandboxBuilder.cs | 82 ++++++- .../core/Extensions.AI/CodeExecutionTool.cs | 4 + .../dotnet/core/PInvoke/SafeNativeMethods.cs | 14 ++ .../CodeExecutionToolTests.cs | 69 ++++++ .../UnikraftBackendTests.cs | 56 +++++ src/sdk/dotnet/ffi/Cargo.toml | 1 + src/sdk/dotnet/ffi/src/lib.rs | 225 +++++++++++++++++- 11 files changed, 503 insertions(+), 16 deletions(-) create mode 100644 src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/UnikraftBackendTests.cs diff --git a/Cargo.lock b/Cargo.lock index 2df2c70..7dda241 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2114,6 +2114,7 @@ dependencies = [ "anyhow", "hyperlight-javascript-sandbox", "hyperlight-sandbox", + "hyperlight-unikraft-sandbox", "hyperlight-wasm-sandbox", "libc", "log", diff --git a/src/sdk/dotnet/README.md b/src/sdk/dotnet/README.md index 375ac03..75b953c 100644 --- a/src/sdk/dotnet/README.md +++ b/src/sdk/dotnet/README.md @@ -7,7 +7,7 @@ A .NET 8.0 SDK for running code in secure, sandboxed environments using [hyperli - **Secure code execution** — run untrusted code in an isolated sandbox - **Tool dispatch** — register .NET functions callable from guest code via `call_tool()` - **Typed tool registration** — auto-serialize/deserialize with `Func` -- **Two backends** — Wasm (Python/JS guests) and built-in JavaScript (QuickJS) +- **Three backends** — Wasm (Python/JS guests), built-in JavaScript (QuickJS), and Unikraft (full CPython with numpy/pandas in a micro-VM) - **Snapshot/restore** — checkpoint and rewind sandbox state (200x faster warm starts) - **Filesystem isolation** — read-only input dirs, writable output dirs, temp output - **Network allowlists** — per-domain HTTP access control with method filtering @@ -98,6 +98,27 @@ using var sandbox = new SandboxBuilder() .Build(); ``` +### Unikraft Backend + +Run full CPython, with native extensions such as numpy and pandas, from a +hyperlight-unikraft rootfs image (the `agent` image the `hluk` CLI caches, +e.g. `~/.cache/hluk/rootfs/ghcr.io/hyperlight-dev/hyperlight-unikraft/agent/initrd-v0.18.0.cpio`): + +```csharp +using var sandbox = new SandboxBuilder() + .WithBackend(SandboxBackend.Unikraft) + .WithModulePath(rootfsPath) + .Build(); + +var result = sandbox.Run("import numpy; print(numpy.arange(10).sum())"); +``` + +`WithScratchSize("2Gi")` sets the guest memory (heap and stack sizes are +refused for this backend); `WithNativeNetwork()` lets guest sockets reach +the allowed hosts. See the +[repository README](../../../README.md#unikraft-sandbox) for how it differs +from the other backends. + ### Snapshot/Restore Checkpoint sandbox state for fast resets between executions: @@ -250,10 +271,12 @@ just dotnet copilot-sdk-example # Run Copilot SDK example | Method | Description | |--------|-------------| -| `WithModulePath(string)` | Path to `.wasm`/`.aot` guest (required for Wasm) | +| `WithModulePath(string)` | Path to `.wasm`/`.aot` guest (required for Wasm), or the rootfs image (required for Unikraft) | | `WithPythonModule()` | Use the bundled Python guest from `Hyperlight.HyperlightSandbox.Guest.Python` | | `WithJavaScriptModule()` | Use the bundled JavaScript guest from `Hyperlight.HyperlightSandbox.Guest.JavaScript` | -| `WithBackend(SandboxBackend)` | `Wasm` (default) or `JavaScript` | +| `WithBackend(SandboxBackend)` | `Wasm` (default), `JavaScript` or `Unikraft` | +| `WithScratchSize(string\|ulong)` | Unikraft only: guest memory (e.g. `"2Gi"`), instead of heap/stack sizes | +| `WithNativeNetwork()` | Unikraft only: guest sockets may reach `AllowDomain` hosts | | `WithHeapSize(string\|ulong)` | Guest heap size (e.g. `"50Mi"`, default: platform-dependent) | | `WithStackSize(string\|ulong)` | Guest stack size (e.g. `"35Mi"`, default: platform-dependent) | | `WithInputDir(string)` | Read-only `/input` directory | diff --git a/src/sdk/dotnet/core/Api/Sandbox.cs b/src/sdk/dotnet/core/Api/Sandbox.cs index 1a1b98b..34312d5 100644 --- a/src/sdk/dotnet/core/Api/Sandbox.cs +++ b/src/sdk/dotnet/core/Api/Sandbox.cs @@ -50,7 +50,9 @@ internal Sandbox( string? outputDir, bool tempOutput, FilesystemLimitsConfiguration? filesystemLimits, - SandboxBackend backend = SandboxBackend.Wasm) + SandboxBackend backend = SandboxBackend.Wasm, + bool nativeNetwork = false, + ulong scratchSize = 0) { // Pin the module path string for the FFI call duration (null for JS backend). var modulePathPtr = modulePath != null @@ -110,6 +112,18 @@ internal Sandbox( var r = SafeNativeMethods.hyperlight_sandbox_set_temp_output(_handle, true); r.ThrowIfError(); } + + if (scratchSize > 0) + { + var r = SafeNativeMethods.hyperlight_sandbox_set_scratch_size(_handle, scratchSize); + r.ThrowIfError(); + } + + if (nativeNetwork) + { + var r = SafeNativeMethods.hyperlight_sandbox_set_native_network(_handle, true); + r.ThrowIfError(); + } } // ----------------------------------------------------------------------- diff --git a/src/sdk/dotnet/core/Api/SandboxBackend.cs b/src/sdk/dotnet/core/Api/SandboxBackend.cs index 72308c8..8764615 100644 --- a/src/sdk/dotnet/core/Api/SandboxBackend.cs +++ b/src/sdk/dotnet/core/Api/SandboxBackend.cs @@ -17,4 +17,26 @@ public enum SandboxBackend /// Uses an embedded QuickJS runtime — no module path needed. /// JavaScript = 1, + + /// + /// hyperlight-unikraft backend: full CPython in a Unikraft micro-VM. + /// Requires the path to a rootfs image (initrd CPIO), e.g. the published + /// agent image with numpy, pandas, scipy and scikit-learn. + /// Its guest memory is set with ; + /// heap and stack sizes are refused. Tool arguments and + /// results and HTTP bodies of any size are exchanged in pieces, up to + /// 64 MiB held unread on the host at a time. See + /// for guest sockets. + /// Exit codes are those a CPython process exits with, 0 to 255 + /// (sys.exit(-1) is 255), so -1 means the guest itself failed. + /// As with the other backends, output files persist from run to run + /// and restoring a snapshot clears them; files the guest has open are + /// interpreter state, and stay open across runs and through a restore. + /// A run that leaves a link, a + /// special file, or files past the filesystem limits in the output + /// directory fails, and the directory is cleared of every file and link + /// and of the directories made during the run; those it had when the run + /// started are kept. A run is refused if the host left such things there. + /// + Unikraft = 2, } diff --git a/src/sdk/dotnet/core/Api/SandboxBuilder.cs b/src/sdk/dotnet/core/Api/SandboxBuilder.cs index 2e74c17..d131e10 100644 --- a/src/sdk/dotnet/core/Api/SandboxBuilder.cs +++ b/src/sdk/dotnet/core/Api/SandboxBuilder.cs @@ -29,6 +29,8 @@ public sealed class SandboxBuilder private bool _tempOutput; private FilesystemLimitsConfiguration? _filesystemLimits; private SandboxBackend _backend = SandboxBackend.Wasm; + private bool _nativeNetwork; + private ulong _scratchSize; /// /// Gets the backend configured for sandboxes built by this builder. @@ -47,11 +49,13 @@ public SandboxBuilder WithBackend(SandboxBackend backend) } /// - /// Sets the path to the guest module (.wasm or .aot file). - /// Required for , not needed for + /// Sets the path to the guest module (.wasm or .aot file), + /// or for the rootfs image (an + /// initrd CPIO). Required for and + /// ; must not be set for /// . /// - /// Absolute or relative path to the guest module. + /// Absolute or relative path to the guest module or rootfs. /// This builder for chaining. public SandboxBuilder WithModulePath(string path) { @@ -61,7 +65,8 @@ public SandboxBuilder WithModulePath(string path) } /// - /// Sets the guest heap size. + /// Sets the guest heap size. Not for , + /// which takes instead. /// /// /// Size string (e.g. "25Mi", "2Gi") or raw bytes as string. @@ -85,7 +90,7 @@ public SandboxBuilder WithHeapSize(ulong bytes) } /// - /// Sets the guest stack size. + /// Sets the guest stack size. Not for . /// /// /// Size string (e.g. "35Mi") or raw bytes as string. @@ -146,6 +151,45 @@ public SandboxBuilder WithTempOutput(bool enabled = true) return this; } + /// + /// Sets the guest memory of a + /// sandbox, overriding the size its image was tested with (1.5 GiB for + /// agent). At least 1 MiB; whole MiB are used. + /// + /// Size string (e.g. "2Gi") or raw bytes as string. + /// This builder for chaining. + public SandboxBuilder WithScratchSize(string size) + { + _scratchSize = SizeParser.Parse(size); + return this; + } + + /// + /// Sets the guest memory of a + /// sandbox in bytes. + /// + /// Guest memory in bytes. + /// This builder for chaining. + public SandboxBuilder WithScratchSize(ulong bytes) + { + _scratchSize = bytes; + return this; + } + + /// + /// Lets guest sockets (Python's requests, urllib) reach the + /// hosts allowed with AllowDomain, by host name only. Unikraft + /// backend only: scheme, port, path and method filters still apply to + /// http_get / http_post alone. + /// + /// Whether to enable it (default: true). + /// This builder for chaining. + public SandboxBuilder WithNativeNetwork(bool enabled = true) + { + _nativeNetwork = enabled; + return this; + } + /// /// Sets finite logical resource limits for the writable filesystem. /// @@ -232,12 +276,36 @@ public Sandbox Build() "Module path is required for the Wasm backend. Call WithModulePath() before Build()."); } + if (_backend == SandboxBackend.Unikraft && string.IsNullOrWhiteSpace(_modulePath)) + { + throw new InvalidOperationException( + "A rootfs image path is required for the Unikraft backend. Call WithModulePath() before Build()."); + } + if (_backend == SandboxBackend.JavaScript && !string.IsNullOrWhiteSpace(_modulePath)) { throw new InvalidOperationException( "Module path must not be set for the JavaScript backend (it has a built-in runtime)."); } + if (_nativeNetwork && _backend != SandboxBackend.Unikraft) + { + throw new InvalidOperationException( + "Native networking applies only to the Unikraft backend."); + } + + if (_scratchSize > 0 && _backend != SandboxBackend.Unikraft) + { + throw new InvalidOperationException( + "The scratch size applies only to the Unikraft backend; use WithHeapSize."); + } + + if (_backend == SandboxBackend.Unikraft && (_heapSize > 0 || _stackSize > 0)) + { + throw new InvalidOperationException( + "Heap and stack sizes do not apply to the Unikraft backend; use WithScratchSize."); + } + return new Sandbox( _modulePath, _heapSize, @@ -246,7 +314,9 @@ public Sandbox Build() _outputDir, _tempOutput, _filesystemLimits, - _backend); + _backend, + _nativeNetwork, + _scratchSize); } } diff --git a/src/sdk/dotnet/core/Extensions.AI/CodeExecutionTool.cs b/src/sdk/dotnet/core/Extensions.AI/CodeExecutionTool.cs index ed5e20b..514f8d2 100644 --- a/src/sdk/dotnet/core/Extensions.AI/CodeExecutionTool.cs +++ b/src/sdk/dotnet/core/Extensions.AI/CodeExecutionTool.cs @@ -40,6 +40,9 @@ public sealed class CodeExecutionTool : IDisposable private const string WasmInitializationCode = "None"; private const string JavaScriptInitializationCode = "void 0;"; + // The Unikraft guest runs CPython, like the Wasm Python guest. + private const string UnikraftInitializationCode = "None"; + private readonly Api.Sandbox _sandbox; private readonly string _initializationCode; private Api.SandboxSnapshot? _snapshot; @@ -231,6 +234,7 @@ public void Dispose() { Api.SandboxBackend.Wasm => WasmInitializationCode, Api.SandboxBackend.JavaScript => JavaScriptInitializationCode, + Api.SandboxBackend.Unikraft => UnikraftInitializationCode, _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, "Unknown sandbox backend."), }; } diff --git a/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs b/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs index 7e005cf..542cf5c 100644 --- a/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs +++ b/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs @@ -169,6 +169,20 @@ internal static partial FFIResult hyperlight_sandbox_set_temp_output( SandboxSafeHandle handle, [MarshalAs(UnmanagedType.I1)] bool enabled); + /// Sets the guest memory in bytes (Unikraft only). + [LibraryImport(LibName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial FFIResult hyperlight_sandbox_set_scratch_size( + SandboxSafeHandle handle, + ulong bytes); + + /// Lets guest sockets reach the allowed hosts (Unikraft only). + [LibraryImport(LibName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial FFIResult hyperlight_sandbox_set_native_network( + SandboxSafeHandle handle, + [MarshalAs(UnmanagedType.I1)] bool enabled); + /// /// Sets the writable filesystem quota policy. /// Mode 0 is finite and mode 1 is unlimited. diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs index 566c07e..04b8084 100644 --- a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs @@ -34,6 +34,74 @@ public void Create_WithJavaScriptBackend_Succeeds() Assert.NotNull(tool); } + [Fact] + public void Create_WithUnikraftBackend_Succeeds() + { + // The rootfs is only opened when the sandbox first runs. + using var tool = new CodeExecutionTool( + new SandboxBuilder() + .WithBackend(SandboxBackend.Unikraft) + .WithModulePath("/tmp/test-rootfs.cpio")); + + Assert.NotNull(tool); + } + + [Fact] + public void Execute_WithUnikraftBackend_RunsPython() + { + // Needs KVM (or MSHV / WHP) and the python image the hluk CLI + // caches; without either there is nothing to run. + var rootfs = CachedUnikraftRootfs("python"); + if (rootfs is null) + { + return; + } + + using var tool = new CodeExecutionTool( + new SandboxBuilder() + .WithBackend(SandboxBackend.Unikraft) + .WithModulePath(rootfs)); + tool.RegisterTool("add", (string json) => "5"); + + ExecutionResult first; + try + { + first = tool.Execute("x = 1\nprint(call_tool('add', a=2, b=3))"); + } + catch (Exception e) when (e.ToString().Contains("No Hypervisor was found")) + { + // Hyperlight's own word that this machine has none it can use; + // any other failure to boot still fails the test. + return; + } + Assert.Equal(0, first.ExitCode); + Assert.Equal("5", first.Stdout.Trim()); + + // Each execution starts from the initialized snapshot. + var second = tool.Execute("print('x' in globals())"); + Assert.Equal("False", second.Stdout.Trim()); + } + + private static string? CachedUnikraftRootfs(string runtime) + { + // The hluk CLI's cache: $HLUK_CACHE_DIR, else %LOCALAPPDATA%\hluk\cache + // on Windows, else $XDG_CACHE_HOME/hluk or ~/.cache/hluk. + var cache = Environment.GetEnvironmentVariable("HLUK_CACHE_DIR"); + if (string.IsNullOrEmpty(cache)) + { + var xdg = Environment.GetEnvironmentVariable("XDG_CACHE_HOME"); + cache = OperatingSystem.IsWindows() + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "hluk", "cache") + : !string.IsNullOrEmpty(xdg) + ? Path.Combine(xdg, "hluk") + : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".cache", "hluk"); + } + + var path = Path.Combine( + cache, "rootfs", "ghcr.io", "hyperlight-dev", "hyperlight-unikraft", runtime, "initrd-v0.18.0.cpio"); + return File.Exists(path) ? path : null; + } + [Fact] public void Create_NullBuilder_ThrowsArgumentNullException() { @@ -63,6 +131,7 @@ public void Execute_AfterDispose_ThrowsObjectDisposedException() [Theory] [InlineData(SandboxBackend.Wasm, "None")] [InlineData(SandboxBackend.JavaScript, "void 0;")] + [InlineData(SandboxBackend.Unikraft, "None")] public void InitializationCodeFor_UsesBackendNoOp( SandboxBackend backend, string expectedCode) diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/UnikraftBackendTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/UnikraftBackendTests.cs new file mode 100644 index 0000000..92ec52a --- /dev/null +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/UnikraftBackendTests.cs @@ -0,0 +1,56 @@ +using HyperlightSandbox.Api; +using Xunit; + +namespace HyperlightSandbox.Tests; + +/// +/// Builder validation for the Unikraft backend (SandboxBackend.Unikraft). +/// None of these boot a guest: the rootfs is opened on the first run. +/// +[System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1515:Consider making public types internal", Justification = "Test classes must be public for xUnit")] +public class UnikraftBackendTests +{ + private static SandboxBuilder Unikraft() => new SandboxBuilder() + .WithBackend(SandboxBackend.Unikraft) + .WithModulePath("/tmp/test-rootfs.cpio"); + + [Fact] + public void WithScratchSize_Builds() + { + using var sandbox = Unikraft().WithScratchSize("512Mi").Build(); + Assert.NotNull(sandbox); + } + + [Fact] + public void WithScratchSize_BelowOneMiB_Throws() + { + Assert.Throws(() => Unikraft().WithScratchSize(512 * 1024).Build()); + } + + [Fact] + public void WithHeapSize_Throws() + { + var error = Assert.Throws(() => Unikraft().WithHeapSize("64Mi").Build()); + Assert.Contains("WithScratchSize", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void WithStackSize_Throws() + { + Assert.Throws(() => Unikraft().WithStackSize("1Mi").Build()); + } + + [Fact] + public void WithScratchSize_OnOtherBackends_Throws() + { + Assert.Throws(() => + new SandboxBuilder().WithBackend(SandboxBackend.JavaScript).WithScratchSize("64Mi").Build()); + } + + [Fact] + public void WithModulePath_IsRequired() + { + Assert.Throws(() => + new SandboxBuilder().WithBackend(SandboxBackend.Unikraft).Build()); + } +} diff --git a/src/sdk/dotnet/ffi/Cargo.toml b/src/sdk/dotnet/ffi/Cargo.toml index b0cbe8c..f2d730e 100644 --- a/src/sdk/dotnet/ffi/Cargo.toml +++ b/src/sdk/dotnet/ffi/Cargo.toml @@ -16,6 +16,7 @@ crate-type = ["cdylib", "rlib"] hyperlight-sandbox.workspace = true hyperlight-wasm-sandbox.workspace = true hyperlight-javascript-sandbox.workspace = true +hyperlight-unikraft-sandbox.workspace = true anyhow = "1" libc = "0.2" log = "0.4" diff --git a/src/sdk/dotnet/ffi/src/lib.rs b/src/sdk/dotnet/ffi/src/lib.rs index 79f498c..858d170 100644 --- a/src/sdk/dotnet/ffi/src/lib.rs +++ b/src/sdk/dotnet/ffi/src/lib.rs @@ -29,6 +29,7 @@ use hyperlight_sandbox::{ DEFAULT_HEAP_SIZE, DEFAULT_STACK_SIZE, DirPerms, FilePerms, FilesystemLimits, GuestSandbox, HttpMethod, Sandbox, SandboxBuilder, SandboxConfig, ToolRegistry, ToolSchema, }; +use hyperlight_unikraft_sandbox::Unikraft; use hyperlight_wasm_sandbox::Wasm; use log::{debug, error}; @@ -113,14 +114,18 @@ impl FFIResult { /// Zero values mean "use platform default". #[repr(C)] pub struct FFISandboxOptions { - /// Path to the `.wasm` or `.aot` guest module (UTF-8, null-terminated). - /// Required for Wasm backend. Must be null for JavaScript backend. + /// Path to the `.wasm` or `.aot` guest module, or for the Unikraft + /// backend the rootfs image (UTF-8, null-terminated). + /// Required for Wasm and Unikraft. Must be null for JavaScript backend. pub module_path: *const c_char, - /// Guest heap size in bytes. 0 = platform default. + /// Guest heap size in bytes. 0 = platform default. Must be 0 for the + /// Unikraft backend, whose memory is set with + /// `hyperlight_sandbox_set_scratch_size`. pub heap_size: u64, - /// Guest stack size in bytes. 0 = platform default. + /// Guest stack size in bytes. 0 = platform default. Must be 0 for the + /// Unikraft backend. pub stack_size: u64, - /// Backend type: 0 = Wasm (default), 1 = JavaScript. + /// Backend type: 0 = Wasm (default), 1 = JavaScript, 2 = Unikraft. pub backend: u32, } @@ -134,6 +139,8 @@ pub enum FFIBackend { Wasm = 0, /// Hyperlight-JS built-in JavaScript backend (no module path needed). JavaScript = 1, + /// hyperlight-unikraft micro-VM running full CPython from a rootfs image. + Unikraft = 2, } /// Filesystem limit policy discriminant. @@ -177,16 +184,24 @@ type JsSnapshotInner = hyperlight_sandbox::Snapshot< <::Sandbox as GuestSandbox>::SnapshotData, >; +type UnikraftSandboxInner = Sandbox; +type UnikraftSnapshotInner = hyperlight_sandbox::Snapshot< + <::Sandbox as GuestSandbox>::SnapshotData, +>; + /// Holds the active backend sandbox instance. enum BackendSandbox { Wasm(WasmSandboxInner), Js(JsSandboxInner), + // Boxed: the micro-VM handle is several times the size of the others. + Unikraft(Box), } /// Holds a snapshot from either backend. enum BackendSnapshot { Wasm(WasmSnapshotInner), Js(JsSnapshotInner), + Unikraft(UnikraftSnapshotInner), } /// Dispatch on the active backend, binding the inner sandbox to `$sb`. @@ -196,6 +211,7 @@ macro_rules! with_sandbox { match $backend { BackendSandbox::Wasm($sb) => $body, BackendSandbox::Js($sb) => $body, + BackendSandbox::Unikraft($sb) => $body, } }; } @@ -215,6 +231,8 @@ struct SandboxState { inner: Option, /// Which backend to use. backend: FFIBackend, + /// Unikraft guest settings (used by the Unikraft backend only). + unikraft: Unikraft, /// Tool callbacks registered before the first `run()`. tools: HashMap, /// Network allowlist entries queued before the sandbox is built. @@ -581,11 +599,12 @@ pub unsafe extern "C" fn hyperlight_sandbox_create(options: FFISandboxOptions) - let backend = match options.backend { 0 => FFIBackend::Wasm, 1 => FFIBackend::JavaScript, + 2 => FFIBackend::Unikraft, other => { return FFIResult::error( FFIErrorCode::InvalidArgument, safe_cstring(format!( - "Invalid backend value: {other}. Use 0 (Wasm) or 1 (JavaScript)." + "Invalid backend value: {other}. Use 0 (Wasm), 1 (JavaScript) or 2 (Unikraft)." )), ); } @@ -608,6 +627,12 @@ pub unsafe extern "C" fn hyperlight_sandbox_create(options: FFISandboxOptions) - safe_cstring("module_path is required for Wasm backend"), ); } + FFIBackend::Unikraft if module_path.is_empty() => { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("module_path (a rootfs image) is required for Unikraft backend"), + ); + } FFIBackend::JavaScript if !module_path.is_empty() => { return FFIResult::error( FFIErrorCode::InvalidArgument, @@ -631,9 +656,24 @@ pub unsafe extern "C" fn hyperlight_sandbox_create(options: FFISandboxOptions) - DEFAULT_STACK_SIZE }; + // A unikernel has one memory size, set with + // `hyperlight_sandbox_set_scratch_size`; as in the Python SDK, heap and + // stack sizes are refused rather than taken to mean it. + if backend == FFIBackend::Unikraft && (options.heap_size > 0 || options.stack_size > 0) { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring( + "heap_size and stack_size do not apply to the Unikraft backend; \ + set the guest memory with the scratch size", + ), + ); + } + let unikraft = Unikraft::new(); + let state = SandboxState { inner: None, backend, + unikraft, tools: HashMap::new(), pending_networks: Vec::new(), config: SandboxConfig { @@ -765,6 +805,80 @@ pub unsafe extern "C" fn hyperlight_sandbox_set_temp_output( FFIResult::success_null() } +/// Lets guest sockets reach the `allow_domain` hosts directly (Unikraft +/// backend only; see `Unikraft::native_network`). +/// +/// Must be called before the first `run()`. +/// +/// # Safety +/// +/// `handle` must be a valid sandbox handle. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn hyperlight_sandbox_set_native_network( + handle: *mut SandboxState, + enabled: bool, +) -> FFIResult { + let state = match unsafe { deref_handle_mut(handle, "sandbox") } { + Ok(s) => s, + Err(e) => return e, + }; + if state.inner.is_some() { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("Cannot set native_network after sandbox has been initialized"), + ); + } + if state.backend != FFIBackend::Unikraft { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("native_network applies only to the Unikraft backend"), + ); + } + state.unikraft = state.unikraft.native_network(enabled); + FFIResult::success_null() +} + +/// Sets the guest memory of a Unikraft sandbox in bytes, overriding the +/// image's tested size (Unikraft backend only; at least 1 MiB, rounded down +/// to whole MiB). +/// +/// Must be called before the first `run()`. +/// +/// # Safety +/// +/// `handle` must be a valid sandbox handle. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn hyperlight_sandbox_set_scratch_size( + handle: *mut SandboxState, + bytes: u64, +) -> FFIResult { + let state = match unsafe { deref_handle_mut(handle, "sandbox") } { + Ok(s) => s, + Err(e) => return e, + }; + if state.inner.is_some() { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("Cannot set scratch_size after sandbox has been initialized"), + ); + } + if state.backend != FFIBackend::Unikraft { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("scratch_size applies only to the Unikraft backend"), + ); + } + let mb = usize::try_from(bytes / (1024 * 1024)).unwrap_or(usize::MAX); + if mb == 0 { + return FFIResult::error( + FFIErrorCode::InvalidArgument, + safe_cstring("scratch_size must be at least 1 MiB"), + ); + } + state.unikraft = state.unikraft.scratch_mb(mb); + FFIResult::success_null() +} + /// Sets logical resource limits for the writable filesystem. /// /// Must be called before the first `run()`. A call replaces the entire @@ -1057,6 +1171,33 @@ fn ensure_initialized(state: &mut SandboxState) -> Result<()> { } BackendSandbox::Js(sb) } + FFIBackend::Unikraft => { + let mut builder = SandboxBuilder::new() + .module_path(&state.config.module_path) + .filesystem_limits(state.filesystem_limits) + .with_tools(registry) + .guest(state.unikraft); + + if let Some(ref dir) = state.input_dir { + builder = builder.input_dir(dir); + } + if let Some(ref dir) = state.output_dir { + builder = builder.output_dir( + dir, + DirPerms::READ | DirPerms::MUTATE, + FilePerms::READ | FilePerms::WRITE, + ); + } else if state.temp_output { + builder = builder.temp_output(); + } + + let mut sb = builder.build()?; + for (target, methods) in std::mem::take(&mut state.pending_networks) { + let method_filter = HttpMethod::parse_list(methods)?; + sb.allow_domain(&target, method_filter)?; + } + BackendSandbox::Unikraft(Box::new(sb)) + } }; state.inner = Some(sandbox); @@ -1254,6 +1395,7 @@ pub unsafe extern "C" fn hyperlight_sandbox_snapshot(handle: *mut SandboxState) let snapshot_result = match sandbox { BackendSandbox::Wasm(sb) => sb.snapshot().map(BackendSnapshot::Wasm), BackendSandbox::Js(sb) => sb.snapshot().map(BackendSnapshot::Js), + BackendSandbox::Unikraft(sb) => sb.snapshot().map(BackendSnapshot::Unikraft), }; match snapshot_result { @@ -1297,6 +1439,7 @@ pub unsafe extern "C" fn hyperlight_sandbox_restore( let result = match (sandbox, snapshot_ref) { (BackendSandbox::Wasm(sb), BackendSnapshot::Wasm(snap)) => sb.restore(snap), (BackendSandbox::Js(sb), BackendSnapshot::Js(snap)) => sb.restore(snap), + (BackendSandbox::Unikraft(sb), BackendSnapshot::Unikraft(snap)) => sb.restore(snap), _ => { return FFIResult::error( FFIErrorCode::InvalidArgument, @@ -1837,6 +1980,76 @@ mod tests { unsafe { hyperlight_sandbox_free_string(result.value) }; } + // ----------------------------------------------------------------------- + // Configuration: set_native_network + // ----------------------------------------------------------------------- + + #[test] + fn unikraft_refuses_heap_and_stack_sizes() { + let path = cstr("/tmp/test-rootfs.cpio"); + for (heap_size, stack_size) in [(64 << 20, 0), (0, 1 << 20)] { + let options = FFISandboxOptions { + module_path: path.as_ptr(), + heap_size, + stack_size, + backend: FFIBackend::Unikraft as u32, + }; + let result = unsafe { hyperlight_sandbox_create(options) }; + assert!(!result.is_success); + assert_eq!(result.error_code, FFIErrorCode::InvalidArgument as u32); + unsafe { hyperlight_sandbox_free_string(result.value) }; + } + } + + #[test] + fn scratch_size_is_for_unikraft_only_and_at_least_one_mib() { + let wasm = create_test_handle(); + let result = unsafe { hyperlight_sandbox_set_scratch_size(wasm, 64 << 20) }; + assert!(!result.is_success); + unsafe { hyperlight_sandbox_free_string(result.value) }; + unsafe { hyperlight_sandbox_free(wasm) }; + + let path = cstr("/tmp/test-rootfs.cpio"); + let options = FFISandboxOptions { + module_path: path.as_ptr(), + heap_size: 0, + stack_size: 0, + backend: FFIBackend::Unikraft as u32, + }; + let created = unsafe { hyperlight_sandbox_create(options) }; + assert!(created.is_success); + let handle = created.value as *mut SandboxState; + let result = unsafe { hyperlight_sandbox_set_scratch_size(handle, 512 * 1024) }; + assert!(!result.is_success); + unsafe { hyperlight_sandbox_free_string(result.value) }; + let result = unsafe { hyperlight_sandbox_set_scratch_size(handle, 2 << 30) }; + assert!(result.is_success); + unsafe { hyperlight_sandbox_free(handle) }; + } + + #[test] + fn set_native_network_is_for_unikraft_only() { + let wasm = create_test_handle(); + let result = unsafe { hyperlight_sandbox_set_native_network(wasm, true) }; + assert!(!result.is_success); + unsafe { hyperlight_sandbox_free_string(result.value) }; + unsafe { hyperlight_sandbox_free(wasm) }; + + let path = cstr("/tmp/test-rootfs.cpio"); + let options = FFISandboxOptions { + module_path: path.as_ptr(), + heap_size: 0, + stack_size: 0, + backend: FFIBackend::Unikraft as u32, + }; + let created = unsafe { hyperlight_sandbox_create(options) }; + assert!(created.is_success); + let handle = created.value as *mut SandboxState; + let result = unsafe { hyperlight_sandbox_set_native_network(handle, true) }; + assert!(result.is_success); + unsafe { hyperlight_sandbox_free(handle) }; + } + // ----------------------------------------------------------------------- // Configuration: set_filesystem_limits // ----------------------------------------------------------------------- From 369e70c4fb432483b30dc2715272532857a92272 Mon Sep 17 00:00:00 2001 From: danbugs Date: Sat, 17 Oct 2026 00:15:13 +0000 Subject: [PATCH 06/10] Document the Unikraft sandbox backend How to run it from Rust and Python, the images it takes, and how it differs from the other backends: one guest memory size, host-mounted /input and /output checked after each run and cleared before it, socket-level networking by host name only, data exchanged in pieces past the 64 KiB host call limit, CPython exit statuses, and hyperlight-unikraft's guest limits. Signed-off-by: danbugs --- README.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.md b/README.md index a6860fa..66025b2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Supported backends: - [Wasm Component Sandbox](#wasm-component-sandbox) (Python/Javascript or provide your own) - [HyperlightJS Sandbox](#hyperlightjs-sandbox) +- [Unikraft Sandbox](#unikraft-sandbox) (full CPython, with numpy/pandas/scipy/scikit-learn) ## Overview @@ -195,6 +196,50 @@ console.log('10 + 20 = ' + sum); See [examples](./src/javascript_sandbox/examples/) for file I/O and network demos. +### Unikraft Sandbox + +Runs full CPython 3.12 in a [Unikraft](https://unikraft.org) unikernel inside a Hyperlight micro-VM, via [hyperlight-unikraft](https://github.com/hyperlight-dev/hyperlight-unikraft). Use it when guest code needs native extensions the Wasm guest cannot load: the published `agent` image ships numpy, pandas, scipy, scikit-learn, matplotlib and more. Guest code gets the same globals as the Wasm Python guest (`call_tool`, `http_get`, `http_post`), `/input` and `/output`, and snapshot/restore of the whole micro-VM. + +The module path is a rootfs image (initrd CPIO). `cached_rootfs("agent")` finds the one the [`hluk`](https://github.com/hyperlight-dev/hyperlight-unikraft) CLI caches (`just unikraft pull-images` pulls `python` and `agent`). + +```rust +use hyperlight_sandbox::{SandboxBuilder, ToolRegistry}; +use hyperlight_unikraft_sandbox::{Unikraft, cached_rootfs}; + +fn main() -> anyhow::Result<()> { + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("fetch_data", |_| Ok(serde_json::json!([1, 2, 3, 4]))); + + let mut sandbox = SandboxBuilder::new() + .module_path(cached_rootfs("agent")?.display().to_string()) + .with_tools(tools) + .guest(Unikraft::new()) + .build()?; + + let result = sandbox.run(r#" +import numpy as np +print(np.mean(call_tool('fetch_data'))) +"#)?; + print!("{}", result.stdout); + Ok(()) +} +``` + +From Python: `Sandbox(backend="unikraft")` (the `agent` image by default; `module="python"` or `module_path=...` for another), installed with `hyperlight-sandbox[unikraft]`. + +How it differs from the other backends: + +- Only the `python`, `python-shell` and `agent` images are supported, and they must be the hyperlight-unikraft release the crate is built against (`IMAGE_RELEASE`). +- Guest memory is one size, the image's tested default (1.5 GiB for `agent`); set it with `Unikraft::scratch_mb` (`scratch_size="2Gi"` in Python, `WithScratchSize("2Gi")` in .NET). Heap and stack sizes do not apply: the Python and .NET SDKs refuse them, and the Rust builder's are ignored. +- `/input` and `/output` are host directories mounted into the guest when it boots, so the guest writes `/output` directly. `/output` is read-write if its permissions allow creating and writing files, else read-only; it cannot be write-only. As with the other backends, files in `/output` persist from run to run, and restoring a snapshot clears them; files the guest has open are interpreter state, so they stay open across runs and through a restore. The filesystem limits are checked after each run: a run that exceeded them, or that left a symbolic or hard link, a special file, or a tree deeper than 64 levels or over 100,000 entries in `/output`, fails, and `/output` is cleared of every file and link (earlier runs' included) and of the directories made during the run; directories it had when the run started are kept, emptied. The bytes did reach the host disk first, so put the output directory on quota-controlled storage. The same check runs before each run, for what the host put in `/output` since: a run that would start on such things is refused, and they are left in place. The guest keeps the directories it booted with; if either is replaced on the host after the sandbox is built, the re-boot an allowlist change makes (with `native_network`) fails until the original is put back. +- `http_get` / `http_post` go through the host and the `allow_domain` allowlist, as in the Wasm guest. Guest sockets (`requests`, `urllib`) are off unless `Unikraft::native_network(true)` (`native_network=True`), and are then allowed to the `allow_domain` hosts by name only: scheme, port, path and method filters apply to `http_get` / `http_post` only. Those names are resolved when the allowlist changes; one that does not resolve is left out (the next run's stderr says so). +- One host call carries at most 64 KiB; larger code, tool arguments and results, and HTTP bodies are chunked transparently. The host holds at most 64 MiB of them unread at a time, so a single tool result or HTTP response can be up to that size. +- Exit statuses are those a CPython process exits with, 0 to 255 (`sys.exit(-1)` is 255, `sys.exit(256)` is 0), so -1 always means the guest itself failed. +- Guest code can run threads and child processes (`subprocess`, and shell pipelines in the `agent` and `python-shell` images, which have a `/bin/sh`). Output written straight to the file descriptors (by child processes, C extensions, or the guest kernel, whose messages are standard error) reaches the run's `stdout` and `stderr` too. +- hyperlight-unikraft's guest limits apply: a single vCPU and no AVX. + +See [examples](./src/unikraft_sandbox/examples/) and [the Python example](./src/sdk/python/core/examples/unikraft_basics.py). + ### Writable filesystem limits Writable output is bounded by default to 5 MiB per file, 20 MiB of cumulative From 34c2311e93f00be622dbb5441aea29696da354ed Mon Sep 17 00:00:00 2001 From: danbugs Date: Wed, 21 Oct 2026 19:21:25 +0000 Subject: [PATCH 07/10] Build against hyperlight-unikraft 0.19.0 0.19.0 has NetworkPolicy::Filter and FsBackend, which the next commits use to put guest sockets and mounts under the sandbox's own policy (hyperlight-dev/hyperlight-unikraft#179, #181). Its images are v0.19.0. The kernel's messages are its guest log now, not the program's standard error, so the agent image test no longer finds a PATH miss there. Signed-off-by: danbugs --- Cargo.lock | 46 ++++++++++--------- Cargo.toml | 2 +- src/sdk/dotnet/README.md | 2 +- .../CodeExecutionToolTests.cs | 2 +- src/unikraft_sandbox/src/lib.rs | 2 +- src/unikraft_sandbox/tests/python_image.rs | 6 +-- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7dda241..2289465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,7 +316,19 @@ dependencies = [ "cap-primitives 3.4.5", "cap-std 3.4.5", "io-lifetimes 2.0.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "cap-fs-ext" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ff379b70af8e08307a8f65e7040c7301cb4a572538ade16b4984f0da77847f" +dependencies = [ + "cap-primitives 4.0.3", + "cap-std 4.0.3", + "io-lifetimes 3.0.1", + "windows-sys 0.61.2", ] [[package]] @@ -345,7 +357,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.4", "rustix-linux-procfs", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -1284,7 +1296,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1370,7 +1382,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes 2.0.4", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2136,10 +2148,11 @@ dependencies = [ [[package]] name = "hyperlight-unikraft" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b40d40daf67b908d9d6366b9bdb1f21b685179d24acfa4da64ac9d28bbc42" +checksum = "1323f2fe645074a1d71b983a1e2f1f9749873983d789d9a9fe794e559bf3d1cf" dependencies = [ + "cap-fs-ext 4.0.3", "cap-std 4.0.3", "getrandom 0.4.3", "hyperlight-common 0.18.0", @@ -2412,7 +2425,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" dependencies = [ "io-lifetimes 2.0.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3643,7 +3656,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4016,12 +4029,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ "bitflags 2.13.2", - "cap-fs-ext", + "cap-fs-ext 3.4.5", "cap-std 3.4.5", "fd-lock", "io-lifetimes 2.0.4", "rustix 0.38.44", - "windows-sys 0.59.0", + "windows-sys 0.52.0", "winx", ] @@ -4965,7 +4978,7 @@ dependencies = [ "async-trait", "bitflags 2.13.2", "bytes", - "cap-fs-ext", + "cap-fs-ext 3.4.5", "cap-net-ext", "cap-rand", "cap-std 3.4.5", @@ -5251,15 +5264,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" @@ -5441,7 +5445,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags 2.13.2", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 05efcaf..2a2b7eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,5 +34,5 @@ hyperlight-wasm = "0.15.0" # Brings its own hyperlight-host/-common 0.18, apart from the 0.17 above that # hyperlight-wasm and hyperlight-js need: no hyperlight type crosses between # the backends, so the two link side by side. -hyperlight-unikraft = { version = "0.18.0", default-features = false } +hyperlight-unikraft = { version = "0.19.0", default-features = false } pyo3 = { version = "0.29", features = ["extension-module"] } diff --git a/src/sdk/dotnet/README.md b/src/sdk/dotnet/README.md index 75b953c..5c55a71 100644 --- a/src/sdk/dotnet/README.md +++ b/src/sdk/dotnet/README.md @@ -102,7 +102,7 @@ using var sandbox = new SandboxBuilder() Run full CPython, with native extensions such as numpy and pandas, from a hyperlight-unikraft rootfs image (the `agent` image the `hluk` CLI caches, -e.g. `~/.cache/hluk/rootfs/ghcr.io/hyperlight-dev/hyperlight-unikraft/agent/initrd-v0.18.0.cpio`): +e.g. `~/.cache/hluk/rootfs/ghcr.io/hyperlight-dev/hyperlight-unikraft/agent/initrd-v0.19.0.cpio`): ```csharp using var sandbox = new SandboxBuilder() diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs index 04b8084..07d727f 100644 --- a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/CodeExecutionToolTests.cs @@ -98,7 +98,7 @@ public void Execute_WithUnikraftBackend_RunsPython() } var path = Path.Combine( - cache, "rootfs", "ghcr.io", "hyperlight-dev", "hyperlight-unikraft", runtime, "initrd-v0.18.0.cpio"); + cache, "rootfs", "ghcr.io", "hyperlight-dev", "hyperlight-unikraft", runtime, "initrd-v0.19.0.cpio"); return File.Exists(path) ? path : null; } diff --git a/src/unikraft_sandbox/src/lib.rs b/src/unikraft_sandbox/src/lib.rs index 31f0a99..9a238ff 100644 --- a/src/unikraft_sandbox/src/lib.rs +++ b/src/unikraft_sandbox/src/lib.rs @@ -70,7 +70,7 @@ const MAX_PENDING_BLOBS: usize = 256; /// The release of the published rootfs images this backend is built /// against. A snapshot (and the guest kernel's host contract) is tied to /// the `hyperlight-unikraft` release, so the images must match it. -pub const IMAGE_RELEASE: &str = "v0.18.0"; +pub const IMAGE_RELEASE: &str = "v0.19.0"; /// The path where the `hluk` CLI caches the published rootfs image of /// `runtime` (`python`, `python-shell`, `agent`, ...) for [`IMAGE_RELEASE`]. diff --git a/src/unikraft_sandbox/tests/python_image.rs b/src/unikraft_sandbox/tests/python_image.rs index c458fa2..62ab1e8 100644 --- a/src/unikraft_sandbox/tests/python_image.rs +++ b/src/unikraft_sandbox/tests/python_image.rs @@ -277,9 +277,9 @@ fn agent_image_runs_child_processes_and_threads() { .unwrap(); assert_eq!(r.exit_code, 0, "{}", r.stderr); assert_eq!(r.stdout, "child\n3\nthread\nkilled\n"); - // The kernel logs each PATH entry a program is not found in; that is - // standard error, kept out of stdout. - assert!(r.stderr.contains("/usr/bin/sleep"), "{}", r.stderr); + // The kernel's log of each PATH entry a program is not found in is the + // guest log, not the program's standard error. + assert!(!r.stderr.contains("ERR:"), "{}", r.stderr); } #[test] From 0bd3a48fe289a17f2e14cde65992447937ade376 Mon Sep 17 00:00:00 2001 From: danbugs Date: Wed, 21 Oct 2026 19:21:25 +0000 Subject: [PATCH 08/10] Check Unikraft guest sockets against the live allowlist With native_network, guest sockets were allowed to the allow_domain hosts by name, resolved into an AllowList at boot, and an allowlist change re-booted the guest from a snapshot to take effect. A name that did not resolve then was left out until the next change. hyperlight-unikraft's NetworkPolicy::Filter asks the embedder about each DNS question and connect instead, so the guest's sockets are checked against the NetworkPermissions http_get uses, as they are at that moment: a TCP connection to an entry's host on its port (the one the URL names, else its scheme's). An allow_domain applies from the next connect, nothing is resolved up front, and the re-boot and its directory checks go. Signed-off-by: danbugs --- README.md | 4 +- src/hyperlight_sandbox/src/network.rs | 2 +- src/sdk/dotnet/core/Api/SandboxBuilder.cs | 4 +- .../core/hyperlight_sandbox/__init__.py | 4 +- src/sdk/python/unikraft_backend/src/lib.rs | 2 +- src/unikraft_sandbox/src/lib.rs | 365 +++++++++++------- src/unikraft_sandbox/tests/python_image.rs | 161 ++++++-- src/unikraft_sandbox/tests/relative_output.rs | 7 +- 8 files changed, 366 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 66025b2..9484415 100644 --- a/README.md +++ b/README.md @@ -231,8 +231,8 @@ How it differs from the other backends: - Only the `python`, `python-shell` and `agent` images are supported, and they must be the hyperlight-unikraft release the crate is built against (`IMAGE_RELEASE`). - Guest memory is one size, the image's tested default (1.5 GiB for `agent`); set it with `Unikraft::scratch_mb` (`scratch_size="2Gi"` in Python, `WithScratchSize("2Gi")` in .NET). Heap and stack sizes do not apply: the Python and .NET SDKs refuse them, and the Rust builder's are ignored. -- `/input` and `/output` are host directories mounted into the guest when it boots, so the guest writes `/output` directly. `/output` is read-write if its permissions allow creating and writing files, else read-only; it cannot be write-only. As with the other backends, files in `/output` persist from run to run, and restoring a snapshot clears them; files the guest has open are interpreter state, so they stay open across runs and through a restore. The filesystem limits are checked after each run: a run that exceeded them, or that left a symbolic or hard link, a special file, or a tree deeper than 64 levels or over 100,000 entries in `/output`, fails, and `/output` is cleared of every file and link (earlier runs' included) and of the directories made during the run; directories it had when the run started are kept, emptied. The bytes did reach the host disk first, so put the output directory on quota-controlled storage. The same check runs before each run, for what the host put in `/output` since: a run that would start on such things is refused, and they are left in place. The guest keeps the directories it booted with; if either is replaced on the host after the sandbox is built, the re-boot an allowlist change makes (with `native_network`) fails until the original is put back. -- `http_get` / `http_post` go through the host and the `allow_domain` allowlist, as in the Wasm guest. Guest sockets (`requests`, `urllib`) are off unless `Unikraft::native_network(true)` (`native_network=True`), and are then allowed to the `allow_domain` hosts by name only: scheme, port, path and method filters apply to `http_get` / `http_post` only. Those names are resolved when the allowlist changes; one that does not resolve is left out (the next run's stderr says so). +- `/input` and `/output` are host directories mounted into the guest when it boots, so the guest writes `/output` directly. `/output` is read-write if its permissions allow creating and writing files, else read-only; it cannot be write-only. As with the other backends, files in `/output` persist from run to run, and restoring a snapshot clears them; files the guest has open are interpreter state, so they stay open across runs and through a restore. The filesystem limits are checked after each run: a run that exceeded them, or that left a symbolic or hard link, a special file, or a tree deeper than 64 levels or over 100,000 entries in `/output`, fails, and `/output` is cleared of every file and link (earlier runs' included) and of the directories made during the run; directories it had when the run started are kept, emptied. The bytes did reach the host disk first, so put the output directory on quota-controlled storage. The same check runs before each run, for what the host put in `/output` since: a run that would start on such things is refused, and they are left in place. The guest keeps the directories it booted with, which are the ones the filesystem limits are checked on: replacing either path on the host afterwards changes nothing for the sandbox. +- `http_get` / `http_post` go through the host and the `allow_domain` allowlist, as in the Wasm guest. Guest sockets (`requests`, `urllib`) are off unless `Unikraft::native_network(true)` (`native_network=True`), and may then make TCP connections to an `allow_domain` entry's host on its port (the one the URL names, else its scheme's: 443 for `https`). The host checks every DNS question and connect against the allowlist as it is at that moment, so an `allow_domain` made later applies from the next connect. The host sees a connection's address and port, and recognises an allowed name's addresses through the answers DNS gives the guest, but not what goes over the connection: another site served from the same address (a CDN, a shared host) is reachable on that port too, and an entry's path and method filters apply to `http_get` / `http_post` only. An entry that allows no method opens no sockets. - One host call carries at most 64 KiB; larger code, tool arguments and results, and HTTP bodies are chunked transparently. The host holds at most 64 MiB of them unread at a time, so a single tool result or HTTP response can be up to that size. - Exit statuses are those a CPython process exits with, 0 to 255 (`sys.exit(-1)` is 255, `sys.exit(256)` is 0), so -1 always means the guest itself failed. - Guest code can run threads and child processes (`subprocess`, and shell pipelines in the `agent` and `python-shell` images, which have a `/bin/sh`). Output written straight to the file descriptors (by child processes, C extensions, or the guest kernel, whose messages are standard error) reaches the run's `stdout` and `stderr` too. diff --git a/src/hyperlight_sandbox/src/network.rs b/src/hyperlight_sandbox/src/network.rs index 56f6292..0200ff8 100644 --- a/src/hyperlight_sandbox/src/network.rs +++ b/src/hyperlight_sandbox/src/network.rs @@ -182,7 +182,7 @@ impl NetworkPermissions { /// The permissions granted so far, in the order they were added. /// /// Backends that enforce the allowlist below the HTTP layer (on raw - /// sockets, by host name) derive their own policy from these. + /// sockets, by host and port) derive their own policy from these. pub fn permissions(&self) -> &[NetworkPermission] { &self.permissions } diff --git a/src/sdk/dotnet/core/Api/SandboxBuilder.cs b/src/sdk/dotnet/core/Api/SandboxBuilder.cs index d131e10..b640667 100644 --- a/src/sdk/dotnet/core/Api/SandboxBuilder.cs +++ b/src/sdk/dotnet/core/Api/SandboxBuilder.cs @@ -178,8 +178,8 @@ public SandboxBuilder WithScratchSize(ulong bytes) /// /// Lets guest sockets (Python's requests, urllib) reach the - /// hosts allowed with AllowDomain, by host name only. Unikraft - /// backend only: scheme, port, path and method filters still apply to + /// hosts allowed with AllowDomain, over TCP on each entry's port. + /// Unikraft backend only: path and method filters still apply to /// http_get / http_post alone. /// /// Whether to enable it (default: true). diff --git a/src/sdk/python/core/hyperlight_sandbox/__init__.py b/src/sdk/python/core/hyperlight_sandbox/__init__.py index 5bd8dcf..ea9e77a 100644 --- a/src/sdk/python/core/hyperlight_sandbox/__init__.py +++ b/src/sdk/python/core/hyperlight_sandbox/__init__.py @@ -134,8 +134,8 @@ class Sandbox: and ``stack_size`` are refused for it: ``scratch_size`` (e.g. ``"2Gi"``) overrides the guest memory the image was tested with. ``native_network`` also lets guest sockets (``requests``, ``urllib``) reach the domains passed - to ``allow_domain``, enforced by host name only; a name that does not - resolve is left out, and the next run's stderr says so. ``/input`` and + to ``allow_domain`` over TCP, on each entry's port (path and method + filters apply to ``http_get`` / ``http_post`` only). ``/input`` and ``/output`` are host directories mounted at boot. As with the other backends, files in ``/output`` persist from run to run and ``restore()`` clears them; open guest files are interpreter state, and stay open too. diff --git a/src/sdk/python/unikraft_backend/src/lib.rs b/src/sdk/python/unikraft_backend/src/lib.rs index c989795..16ec4e0 100644 --- a/src/sdk/python/unikraft_backend/src/lib.rs +++ b/src/sdk/python/unikraft_backend/src/lib.rs @@ -40,7 +40,7 @@ impl UnikraftSandbox { /// published image named by `runtime` (default `agent`) is taken from /// the `hluk` CLI's cache. `scratch_size` overrides the guest memory /// the image was tested with (e.g. `"2Gi"`). `native_network` also lets - /// guest sockets reach the allowed domains (by host name only). + /// guest sockets reach the allowed domains, over TCP on their ports. #[new] #[pyo3(signature = (input_dir=None, output_dir=None, temp_output=false, module_path=None, runtime=None, scratch_size=None, native_network=false, filesystem_limits=None, max_file_size=None, max_total_size=None, max_file_count=None))] fn new( diff --git a/src/unikraft_sandbox/src/lib.rs b/src/unikraft_sandbox/src/lib.rs index 9a238ff..e406b83 100644 --- a/src/unikraft_sandbox/src/lib.rs +++ b/src/unikraft_sandbox/src/lib.rs @@ -12,7 +12,7 @@ //! | `call_tool(name, **kwargs)` | host function `sandbox.call_tool` | //! | `http_get` / `http_post` | host function `sandbox.http` (allowlisted) | //! | `/input`, `/output` | hostfs mounts (read-only / read-write) | -//! | `allow_domain` | checked per HTTP request on the host; raw sockets opt-in via [`Unikraft::native_network`] | +//! | `allow_domain` | checked per HTTP request and, opt-in via [`Unikraft::native_network`], per socket connect | //! | `snapshot` / `restore` | hyperlight snapshots of the whole micro-VM | //! //! The module path given to the sandbox builder is the rootfs image (an @@ -22,6 +22,7 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -30,13 +31,13 @@ use base64::Engine as _; use cap_std::fs::Dir as CapDir; use hyperlight_sandbox::runtime::BlockOn; use hyperlight_sandbox::{ - CapFs, DirPerms, ExecutionResult, FilePerms, Guest, GuestSandbox, HttpMethod, + CapFs, DirPerms, ExecutionResult, FilePerms, Guest, GuestSandbox, HttpMethod, MethodFilter, NetworkPermissions, SandboxConfig, Snapshot, ToolRegistry, http as sandbox_http, }; pub use hyperlight_unikraft; use hyperlight_unikraft::{ - AllowList, AppSandbox, Mount, NetworkPolicy, SandboxBuilder as HlukBuilder, - Snapshot as HlukSnapshot, + AppSandbox, Destination, Filter, Mount, NetworkFilter, NetworkPolicy, + SandboxBuilder as HlukBuilder, Snapshot as HlukSnapshot, }; /// The guest code that defines `call_tool`, `http_get`, `http_post` and the @@ -131,10 +132,8 @@ pub fn cached_rootfs(runtime: &str) -> Result { /// `/output` is cleared of every file and link and of the directories made /// during the run; those it had when the run started are kept, emptied. A /// run that would start on such things is refused, and they are left as -/// they are. The guest is (re)booted -/// only onto the directories the sandbox was built with: one replaced on -/// the host since then fails the boot an allowlist change makes, until it -/// is put back. +/// they are. The guest is booted only onto +/// the directories the sandbox was built with. /// /// A run's exit status is the one a CPython process would exit with, 0 to /// 255 (`sys.exit(-1)` is 255), so -1 always means the guest failed. @@ -162,13 +161,20 @@ impl Unikraft { /// Also let guest sockets reach the hosts named by `allow_domain`, so /// libraries like `requests` and `urllib` work directly. /// + /// A guest socket may then make a TCP connection to an `allow_domain` + /// entry's host on its port (the one the URL names, else its scheme's, + /// 443 for `https`): the host checks each connect, and each DNS + /// question, against the allowlist as it is at that moment, so an + /// `allow_domain` made later applies from the next connect. + /// /// Off by default because socket-level enforcement is coarser than the - /// HTTP-level allowlist the other backends apply: it is by host name - /// only, so the scheme, port, path prefix and method filters of an - /// `allow_domain` entry are not enforced on these sockets (they still - /// are for `http_get` / `http_post`). The names are resolved when the - /// allowlist changes; one that does not resolve then is left out until - /// it changes again, and the next run's stderr says so. + /// HTTP-level allowlist the other backends apply. The host sees the + /// address and port of a connection, and recognises an allowed name's + /// addresses through the answers DNS gives the guest for it, but not + /// what goes over the connection: another site served from the same + /// address (a CDN, a shared host) is reachable on that port too, and an + /// entry's path prefix and method filters are not enforced on these + /// sockets (they still are for `http_get` / `http_post`). pub fn native_network(mut self, on: bool) -> Self { self.native_network = on; self @@ -437,14 +443,6 @@ fn push_capped(buffer: &mut String, text: &str) { pub struct UnikraftGuestSandbox { sandbox: AppSandbox, host: HostState, - mounts: Vec, - native_network: bool, - /// The hosts the guest's socket policy was built from, to notice an - /// `allow_domain` made after boot. - socket_hosts: Vec, - /// Said in the next run's stderr: which allowed hosts guest sockets - /// cannot reach because they did not resolve. - socket_notice: Option, files: Arc>, /// The directories `/output` had when the current run started, relative /// to it: kept when a failed check clears the run's output, so that @@ -483,8 +481,11 @@ impl UnikraftGuestSandbox { stderr: Arc::default(), }; let mounts = mounts_for(&files)?; - let socket_hosts = socket_hosts(guest.native_network, &network)?; - let (policy, socket_notice) = socket_policy(&socket_hosts); + let policy = guest.native_network.then(|| { + NetworkPolicy::Filter(Filter::new(SocketFilter { + network: network.clone(), + })) + }); let mut builder = HlukBuilder::from_initrd(&rootfs); if let Some(mb) = guest.scratch_mb { @@ -513,43 +514,12 @@ impl UnikraftGuestSandbox { Ok(Self { sandbox, host, - mounts, - native_network: guest.native_network, - socket_hosts, - socket_notice, files, // Taken before each run kept_dirs: HashSet::new(), }) } - /// Re-boot the guest under the current allowlist, keeping its state: - /// the socket policy is fixed at boot, and a snapshot restores under a - /// different one. - fn sync_socket_policy(&mut self) -> Result<()> { - let hosts = socket_hosts(self.native_network, &self.host.network)?; - if hosts == self.socket_hosts { - return Ok(()); - } - // Resolved once per change of the allowlist, not every run: a name - // that does not resolve leaves guest sockets unable to reach it, as - // DNS would, rather than failing every run from then on. - let (policy, notice) = socket_policy(&hosts); - check_mounts(&self.files, &self.mounts)?; - let snapshot = self.sandbox.snapshot()?; - let builder = configure( - HlukBuilder::from_snapshot(snapshot), - &self.host, - &self.mounts, - policy, - ); - self.sandbox = builder.boot()?; - self.sandbox.drain_output(); - self.socket_hosts = hosts; - self.socket_notice = notice; - Ok(()) - } - fn run_impl(&mut self, code: &str) -> Result { { let mut files = self @@ -568,7 +538,6 @@ impl UnikraftGuestSandbox { self.kept_dirs = output_dirs(&files).context("failed to list the directories in /output")?; } - self.sync_socket_policy()?; self.host.take_stderr()?; self.host .blobs @@ -599,10 +568,6 @@ impl UnikraftGuestSandbox { ); push_capped(&mut stderr, &raw_stderr); - if let Some(notice) = self.socket_notice.take() { - stderr.insert_str(0, ¬ice); - } - // Nothing below returns early: the output limits are enforced, and // the output kept, whatever the guest returned. let mut exit_code = match outcome { @@ -773,9 +738,8 @@ fn mounts_for(files: &Mutex) -> Result> { let files = files .lock() .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; - // Resolved now, as `CapFs` resolved them: a re-boot remounts them, - // and a relative path would then follow the process's working - // directory elsewhere. + // Resolved now, as `CapFs` resolved them: a relative path would + // follow the process's working directory, which may change. let absolute = |path: &Path| { std::fs::canonicalize(path).with_context(|| format!("cannot resolve {}", path.display())) }; @@ -794,48 +758,100 @@ fn mounts_for(files: &Mutex) -> Result> { Ok(mounts) } -/// The host names guest sockets may reach: none unless native networking -/// is on, else those of every `allow_domain` entry. -fn socket_hosts(native: bool, network: &Mutex) -> Result> { - if !native { - return Ok(Vec::new()); +/// What guest sockets may reach: TCP connections to the host of an +/// `allow_domain` entry on its port, looked up in the allowlist as it is +/// at each DNS question and connect. A name is recognised by address, +/// through the answers DNS gives the guest for it. +struct SocketFilter { + network: Arc>, +} + +/// An allowlist entry's host, as guest sockets are checked against it. +#[derive(Debug, PartialEq)] +enum SocketHost<'a> { + /// A name, without a trailing dot, in the case the URL gives it (lower + /// for `http` and `https`, as written for a scheme `url` does not know). + Name(&'a str), + /// An address, IPv4-mapped IPv6 as IPv4, as the filter is given it. + Addr(IpAddr), +} + +impl<'a> SocketHost<'a> { + fn of(host: &url::Host<&'a str>) -> Self { + let canonical = |ip: IpAddr| match ip { + IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(ip, IpAddr::V4), + v4 => v4, + }; + match *host { + // A scheme `url` does not know leaves an address as a name. + url::Host::Domain(domain) => domain + .parse() + .map(|ip| Self::Addr(canonical(ip))) + .unwrap_or_else(|_| Self::Name(domain.trim_end_matches('.'))), + url::Host::Ipv4(address) => Self::Addr(IpAddr::V4(address)), + url::Host::Ipv6(address) => Self::Addr(canonical(IpAddr::V6(address))), + } + } + + /// Whether this is `name`, as DNS spells it to the filter: lower case, + /// no trailing dot. + fn is_name(&self, name: &str) -> bool { + matches!(self, Self::Name(n) if n.eq_ignore_ascii_case(name)) + } + + /// Whether this is `d`'s host: one of the names DNS gave for its + /// address, or the address itself. + fn is(&self, d: &Destination<'_>) -> bool { + match self { + Self::Name(_) => d.names.iter().any(|n| self.is_name(n)), + Self::Addr(ip) => *ip == d.addr.ip(), + } } - let network = network - .lock() - .map_err(|_| anyhow::anyhow!("network mutex poisoned"))?; - let mut hosts: Vec = network - .permissions() - .iter() - .filter_map(|p| p.url.host().map(|host| allowlist_host(&host))) - .collect(); - hosts.sort(); - hosts.dedup(); - Ok(hosts) } -/// A URL's host as hyperlight-unikraft's allowlist takes it: a name, or an -/// address without the brackets a URL puts around IPv6. -fn allowlist_host(host: &url::Host<&str>) -> String { - match host { - url::Host::Domain(name) => (*name).to_owned(), - url::Host::Ipv4(address) => address.to_string(), - // As an IPv4 address when it is one: the allowlist compares - // addresses in that form. - url::Host::Ipv6(address) => address - .to_ipv4_mapped() - .map_or_else(|| address.to_string(), |v4| v4.to_string()), +impl SocketFilter { + /// Whether some entry's host and port satisfy `allows`. An entry that + /// allows no HTTP method at all (none, or only the always-refused + /// CONNECT and TRACE) opens no sockets either. + fn any(&self, mut allows: impl FnMut(&SocketHost<'_>, Option) -> bool) -> bool { + // A poisoned allowlist allows nothing. + let Ok(network) = self.network.lock() else { + return false; + }; + network.permissions().iter().any(|p| { + let refuses_all = matches!(&p.methods, MethodFilter::Only(methods) + if methods.iter().all(|m| matches!(m, HttpMethod::Connect | HttpMethod::Trace))); + !refuses_all + && p.url.host().is_some_and(|host| { + allows(&SocketHost::of(&host), p.url.port_or_known_default()) + }) + }) + } +} + +impl NetworkFilter for SocketFilter { + fn allows_name(&self, name: &str) -> bool { + self.any(|host, _| host.is_name(name)) + } + + fn allows(&self, d: &Destination<'_>) -> bool { + // An HTTP allowlist says nothing for UDP (DNS questions to the + // resolvers are let through before the filter is asked). + if d.udp { + return false; + } + self.any(|host, port| port == Some(d.addr.port()) && host.is(d)) } } /// Refuse to mount a directory other than the one `CapFs` opened for it. /// /// hostfs opens each mount by path when the guest boots and keeps that -/// directory until the next boot (a restore keeps it too), while `CapFs` -/// checks and clears the one it opened at build. So this is checked before -/// each boot, the first and the re-boot an allowlist change makes: were the -/// path replaced since the build (`rm -rf out && mkdir out`), the guest -/// would write a directory nothing checks. Between boots, replacing it -/// changes nothing for either side. +/// directory from then on (a restore keeps it too), while `CapFs` checks +/// and clears the one it opened at build. So this is checked before the +/// boot: were the path replaced since the build (`rm -rf out && mkdir +/// out`), the guest would write a directory nothing checks. After the +/// boot, replacing it changes nothing for either side. fn check_mounts(files: &Mutex, mounts: &[Mount]) -> Result<()> { let files = files .lock() @@ -861,46 +877,11 @@ fn check_mounts(files: &Mutex, mounts: &[Mount]) -> Result<()> { Ok(()) } -/// The guest's socket policy for `hosts`, and a note naming those of them -/// that did not resolve, which the policy leaves out. -fn socket_policy(hosts: &[String]) -> (Option, Option) { - if hosts.is_empty() { - return (None, None); - } - let mut reachable: Vec<&String> = hosts - .iter() - .filter(|host| AllowList::from_hosts(&[host.as_str()]).is_ok()) - .collect(); - // Resolved again as one list: drop any that stopped resolving since. - let policy = loop { - if reachable.is_empty() { - break None; - } - match AllowList::from_hosts(&reachable) { - Ok(allow) => break Some(allow), - Err(e) => reachable.retain(|host| host.as_str() != e.host()), - } - }; - let unresolved: Vec<&str> = hosts - .iter() - .filter(|host| !reachable.contains(host)) - .map(String::as_str) - .collect(); - let notice = (!unresolved.is_empty()).then(|| { - format!( - "[hyperlight-sandbox] native network: {} did not resolve; guest sockets cannot \ - reach it until the allowlist changes\n", - unresolved.join(", ") - ) - }); - (policy, notice) -} - fn configure( builder: HlukBuilder, host: &HostState, mounts: &[Mount], - policy: Option, + policy: Option, ) -> HlukBuilder { // Raw guest output belongs in the run's result, not in the embedding // process's stdout, and is held to the same cap as the rest of it. @@ -909,8 +890,8 @@ fn configure( .echo_output(false) .output_limit(MAX_OUTPUT_BYTES) .mounts(mounts.iter().cloned()); - if let Some(allow) = policy { - builder = builder.network(NetworkPolicy::AllowList(allow)); + if let Some(policy) = policy { + builder = builder.network(policy); } builder } @@ -941,7 +922,12 @@ impl GuestSandbox for UnikraftGuestSandbox { #[cfg(test)] mod tests { - use super::{allowlist_host, exit_status, strip_stderr_lines, uploaded}; + use std::sync::{Arc, Mutex}; + + use hyperlight_sandbox::{HttpMethod, MethodFilter, NetworkPermissions}; + use hyperlight_unikraft::{Destination, NetworkFilter}; + + use super::{SocketFilter, exit_status, strip_stderr_lines, uploaded}; #[test] fn exit_statuses_outside_a_byte_are_failures() { @@ -976,15 +962,106 @@ mod tests { assert_eq!(strip_stderr_lines("out err\n", "err\n"), None); } + fn filter(entries: &[&str]) -> SocketFilter { + let mut network = NetworkPermissions::default(); + for entry in entries { + network.allow_domain(entry, MethodFilter::Any).unwrap(); + } + SocketFilter { + network: Arc::new(Mutex::new(network)), + } + } + + fn tcp<'a>(addr: &str, names: &'a [String]) -> Destination<'a> { + Destination::new(addr.parse().unwrap(), false, names) + } + #[test] - fn allowlist_hosts_have_no_ipv6_brackets() { - let host = |url: &str| allowlist_host(&url::Url::parse(url).unwrap().host().unwrap()); - assert_eq!( - host("https://[2606:4700:4700::1111]/"), - "2606:4700:4700::1111" - ); - assert_eq!(host("http://[::ffff:1.2.3.4]/"), "1.2.3.4"); - assert_eq!(host("http://127.0.0.1:8080"), "127.0.0.1"); - assert_eq!(host("https://example.com"), "example.com"); + fn sockets_reach_allowed_names_on_their_port() { + let f = filter(&["https://api.example.com/v1", "http://plain.example:8080"]); + assert!(f.allows_name("api.example.com")); + assert!(f.allows_name("plain.example")); + assert!(!f.allows_name("example.com")); + let api = ["api.example.com".to_owned()]; + assert!(f.allows(&tcp("203.0.113.9:443", &api))); + assert!(!f.allows(&tcp("203.0.113.9:80", &api))); + let plain = ["plain.example".to_owned()]; + assert!(f.allows(&tcp("203.0.113.10:8080", &plain))); + assert!(!f.allows(&tcp("203.0.113.10:80", &plain))); + // An address DNS gave for no allowed name, or for none at all. + assert!(!f.allows(&tcp("203.0.113.9:443", &["evil.example".to_owned()]))); + assert!(!f.allows(&tcp("203.0.113.9:443", &[]))); + // UDP is not HTTP. + assert!(!f.allows(&Destination::new( + "203.0.113.9:443".parse().unwrap(), + true, + &api + ))); + } + + #[test] + fn sockets_reach_allowed_addresses_by_address() { + let f = filter(&[ + "https://[2606:4700:4700::1111]", + "http://[::ffff:1.2.3.4]", + "http://10.0.0.5:8080", + ]); + assert!(f.allows(&tcp("[2606:4700:4700::1111]:443", &[]))); + assert!(f.allows(&tcp("1.2.3.4:80", &[]))); + assert!(f.allows(&tcp("10.0.0.5:8080", &[]))); + assert!(!f.allows(&tcp("10.0.0.5:80", &[]))); + assert!(!f.allows_name("10.0.0.5")); + } + + #[test] + fn hosts_are_compared_as_dns_spells_them() { + // A trailing dot, a scheme `url` leaves alone (case kept, an + // address left as a name). + let f = filter(&[ + "https://example.com.", + "foo://DB.internal:5432", + "foo://1.2.3.4:22", + ]); + assert!(f.allows_name("example.com")); + assert!(f.allows(&tcp("203.0.113.9:443", &["example.com".to_owned()]))); + assert!(f.allows_name("db.internal")); + assert!(f.allows(&tcp("10.0.0.7:5432", &["db.internal".to_owned()]))); + assert!(f.allows(&tcp("1.2.3.4:22", &[]))); + assert!(!f.allows_name("1.2.3.4")); + } + + #[test] + fn entries_that_allow_no_method_open_no_sockets() { + let mut network = NetworkPermissions::default(); + for methods in [vec![], vec![HttpMethod::Connect, HttpMethod::Trace]] { + network + .allow_domain("https://api.example.com", MethodFilter::Only(methods)) + .unwrap(); + } + let f = SocketFilter { + network: Arc::new(Mutex::new(network)), + }; + assert!(!f.allows_name("api.example.com")); + f.network + .lock() + .unwrap() + .allow_domain("https://api.example.com", vec![HttpMethod::Get]) + .unwrap(); + assert!(f.allows_name("api.example.com")); + } + + #[test] + fn sockets_follow_the_allowlist_as_it_changes() { + let f = filter(&[]); + let api = ["api.example.com".to_owned()]; + assert!(!f.allows_name("api.example.com")); + assert!(!f.allows(&tcp("203.0.113.9:443", &api))); + f.network + .lock() + .unwrap() + .allow_domain("https://api.example.com", MethodFilter::Any) + .unwrap(); + assert!(f.allows_name("api.example.com")); + assert!(f.allows(&tcp("203.0.113.9:443", &api))); } } diff --git a/src/unikraft_sandbox/tests/python_image.rs b/src/unikraft_sandbox/tests/python_image.rs index 62ab1e8..bc1066e 100644 --- a/src/unikraft_sandbox/tests/python_image.rs +++ b/src/unikraft_sandbox/tests/python_image.rs @@ -387,33 +387,98 @@ fn symlinks_in_output_are_removed() { assert_eq!(std::fs::read_dir(output).unwrap().count(), 0); } +/// This host's address on the interface it reaches the internet by: not +/// loopback, which guest sockets may never reach. +fn host_ip() -> Option { + let probe = std::net::UdpSocket::bind("0.0.0.0:0").ok()?; + probe.connect("8.8.8.8:53").ok()?; + Some(probe.local_addr().ok()?.ip()) +} + +/// A connection the guest opened stays open when the allowlist changes: +/// the change is checked at the next connect, without booting the guest +/// again (which would drop its sockets). A name that does not resolve is +/// no obstacle either, as nothing is resolved until the guest asks. #[test] -fn unresolvable_hosts_do_not_break_native_network() { +fn native_network_follows_the_allowlist_without_a_reboot() { + use std::io::Read as _; let Some(rootfs) = rootfs("python") else { return; }; + let Some(ip) = host_ip() else { + eprintln!("skipping: no outbound interface"); + return; + }; + let first = std::net::TcpListener::bind((ip, 0)).unwrap(); + let second = std::net::TcpListener::bind((ip, 0)).unwrap(); + let (p1, p2) = ( + first.local_addr().unwrap().port(), + second.local_addr().unwrap().port(), + ); let mut sandbox = SandboxBuilder::new() .module_path(rootfs) .guest(Unikraft::new().native_network(true)) .build() .unwrap(); + let connect = |port: u16| { + format!( + "import errno, socket\n\ + def connect(port):\n \ + s = socket.socket()\n \ + s.settimeout(5)\n \ + try:\n \ + s.connect(('{ip}', port))\n \ + return s\n \ + except OSError as e:\n \ + print('refused' if e.errno == errno.EACCES else repr(e))\n\ + kept = connect({port})\n\ + print('connected' if kept else 'not connected')" + ) + }; + let r = sandbox.run(&connect(p1)).unwrap(); + assert_eq!(r.stdout, "refused\nnot connected\n", "{}", r.stderr); + + sandbox + .allow_domain( + &format!("http://{ip}:{p1}"), + hyperlight_sandbox::MethodFilter::Any, + ) + .unwrap(); + let r = sandbox.run(&connect(p1)).unwrap(); + assert_eq!(r.stdout, "connected\n", "{}", r.stderr); + let (mut held, _) = first.accept().unwrap(); + sandbox .allow_domain( "https://does-not-resolve.invalid", hyperlight_sandbox::MethodFilter::Any, ) .unwrap(); - let r = sandbox.run("print('first')").unwrap(); - assert_eq!((r.exit_code, r.stdout.as_str()), (0, "first\n")); - assert!( - r.stderr.contains("does-not-resolve.invalid"), - "{}", - r.stderr - ); - // Said once, and later runs are unaffected. - let r = sandbox.run("print('second')").unwrap(); - assert_eq!((r.exit_code, r.stdout.as_str()), (0, "second\n")); + sandbox + .allow_domain( + &format!("http://{ip}:{p2}"), + hyperlight_sandbox::MethodFilter::Any, + ) + .unwrap(); + // The connection from before the change still carries data, and the + // newly allowed port is reachable. + let r = sandbox + .run(&format!( + "kept.sendall(b'still open')\n\ + other = connect({p2})\n\ + print('second connected' if other else 'second not connected')" + )) + .unwrap(); + assert_eq!(r.stdout, "second connected\n", "{}", r.stderr); assert!(r.stderr.is_empty(), "{}", r.stderr); + let mut got = [0u8; 10]; + held.set_read_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + held.read_exact(&mut got).unwrap(); + assert_eq!(&got, b"still open"); + // A port on the same host that no entry names is still refused. + let r = sandbox.run(&format!("connect({})", p2 ^ 1)).unwrap(); + assert_eq!(r.stdout, "refused\n", "{}", r.stderr); } #[test] @@ -597,8 +662,57 @@ fn a_run_is_refused_on_what_the_host_put_in_output_past_the_rules() { assert!(output.path().join("ran").is_file()); } +/// Guest sockets reach an allowed name on its scheme's port, by the names +/// DNS gives the guest, and nothing else. Needs the internet; skips +/// without it. #[test] -fn a_replaced_output_directory_is_refused() { +fn native_network_reaches_allowed_names_by_name() { + let Some(rootfs) = rootfs("python") else { + return; + }; + if std::net::ToSocketAddrs::to_socket_addrs(&("example.com", 80)).is_err() { + eprintln!("skipping: example.com does not resolve (no internet?)"); + return; + } + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .guest(Unikraft::new().native_network(true)) + .build() + .unwrap(); + sandbox + .allow_domain("http://example.com", hyperlight_sandbox::MethodFilter::Any) + .unwrap(); + let r = sandbox + .run( + "import errno, socket, urllib.request\n\ + print('allowed', urllib.request.urlopen('http://example.com', timeout=15).status)\n\ + s = socket.socket()\n\ + s.settimeout(5)\n\ + try:\n \ + s.connect(('example.com', 443))\n \ + print('443 connected')\n\ + except OSError as e:\n \ + print('443', 'refused' if e.errno == errno.EACCES else repr(e))\n\ + try:\n \ + socket.getaddrinfo('github.com', 80)\n \ + print('other name resolved')\n\ + except OSError:\n \ + print('other name refused')", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, "allowed 200\n443 refused\nother name refused\n", + "{}", + r.stderr + ); +} + +/// The guest keeps the `/output` it booted onto, which is the directory +/// `CapFs` checks: replacing the path on the host afterwards sends neither +/// side to the new one. +#[test] +fn a_replaced_output_directory_changes_nothing() { let Some(rootfs) = rootfs("python") else { return; }; @@ -612,7 +726,7 @@ fn a_replaced_output_directory_is_refused() { hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, ) - .guest(Unikraft::new().native_network(true)) + .guest(Unikraft::new()) .build() .unwrap(); assert_eq!(sandbox.run("pass").unwrap().exit_code, 0); @@ -621,20 +735,17 @@ fn a_replaced_output_directory_is_refused() { let kept = base.path().join("kept"); std::fs::rename(&output, &kept).unwrap(); std::fs::create_dir(&output).unwrap(); - sandbox - .allow_domain("http://127.0.0.1", hyperlight_sandbox::MethodFilter::Any) + let r = sandbox + .run("open('/output/a.txt', 'w').write('a')") .unwrap(); - let code = "import os\nos.symlink('x', '/output/link')"; - let error = sandbox.run(code).unwrap_err(); - assert!(format!("{error:#}").contains("was replaced"), "{error:#}"); - // Nothing reached the new directory unchecked. + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert!(kept.join("a.txt").is_file()); assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); - - // Putting the original back lets the re-boot, and the run, go ahead. - std::fs::remove_dir(&output).unwrap(); - std::fs::rename(&kept, &output).unwrap(); - let r = sandbox.run(code).unwrap(); + // And what breaks the rules there is caught and cleared there. + let r = sandbox + .run("import os\nos.symlink('x', '/output/link')") + .unwrap(); assert_eq!(r.exit_code, 1, "{}", r.stderr); assert!(r.stderr.contains("symbolic links"), "{}", r.stderr); - assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); + assert_eq!(std::fs::read_dir(&kept).unwrap().count(), 0); } diff --git a/src/unikraft_sandbox/tests/relative_output.rs b/src/unikraft_sandbox/tests/relative_output.rs index 78498c7..e919ca5 100644 --- a/src/unikraft_sandbox/tests/relative_output.rs +++ b/src/unikraft_sandbox/tests/relative_output.rs @@ -36,7 +36,7 @@ fn relative_output_dir_ignores_later_chdir() { DirPerms::READ | DirPerms::MUTATE, FilePerms::READ | FilePerms::WRITE, ) - .guest(Unikraft::new().native_network(true)) + .guest(Unikraft::new()) .build() .unwrap(); std::env::set_current_dir(&b).unwrap(); @@ -58,11 +58,6 @@ fn relative_output_dir_ignores_later_chdir() { std::fs::read_to_string(b.join("out/userdata/precious.txt")).unwrap(), "keep" ); - // A change to the allowlist re-boots the guest, remounting /output: - // still a/out. - sandbox - .allow_domain("http://127.0.0.1", hyperlight_sandbox::MethodFilter::Any) - .unwrap(); let r = sandbox .run("open('/output/y.txt', 'w').write('y')") .unwrap(); From 02c69c04de77238068cfb002975626815debaff7 Mon Sep 17 00:00:00 2001 From: danbugs Date: Wed, 21 Oct 2026 21:23:44 +0000 Subject: [PATCH 09/10] CapFs: path operations for a guest with a filesystem of its own A hyperlight-unikraft guest asks the host for each file operation by path, not through WASI descriptors. stat_path, read_path, write_path, create_dir_path, remove_path, truncate_path, list_path and rename_path serve it by the rules the descriptor operations apply: the directory permissions, and for the output the filesystem limits (checked before the host directory changes, and kept as files are removed, truncated or renamed over), no hard-linked files, and the bounds the output scan walks (64 levels, a directory moved included, and 100,000 entries). Only regular files are read, so a FIFO cannot block the call. Errors are io::Errors, as the guest's file calls should see them. Signed-off-by: danbugs --- Cargo.lock | 1 + src/hyperlight_sandbox/Cargo.toml | 1 + src/hyperlight_sandbox/src/cap_fs.rs | 802 ++++++++++++++++++++++++++- 3 files changed, 794 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2289465..048e30d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2071,6 +2071,7 @@ dependencies = [ "anyhow", "bitflags 2.13.2", "bytes", + "cap-fs-ext 4.0.3", "cap-std 4.0.3", "futures-util", "http", diff --git a/src/hyperlight_sandbox/Cargo.toml b/src/hyperlight_sandbox/Cargo.toml index a73a542..ea91b49 100644 --- a/src/hyperlight_sandbox/Cargo.toml +++ b/src/hyperlight_sandbox/Cargo.toml @@ -13,6 +13,7 @@ description = "High-level Rust host library for running sandbox guests across mu anyhow = "1" bitflags = "2" bytes = "1" +cap-fs-ext = "4.0.3" cap-std = "4.0.3" http = "1" http-body-util = "0.1" diff --git a/src/hyperlight_sandbox/src/cap_fs.rs b/src/hyperlight_sandbox/src/cap_fs.rs index e418210..cd075bc 100644 --- a/src/hyperlight_sandbox/src/cap_fs.rs +++ b/src/hyperlight_sandbox/src/cap_fs.rs @@ -20,11 +20,13 @@ use std::collections::HashMap; use std::fmt; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::{self, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::SystemTime; use anyhow::{Context, Result}; +use cap_fs_ext::{DirExt, SystemTimeSpec}; use cap_std::ambient_authority; use cap_std::fs::Dir as CapDir; @@ -65,6 +67,22 @@ impl fmt::Display for FsError { impl std::error::Error for FsError {} +/// As a guest's own file calls would see it, for [`CapFs`]'s path +/// operations. +impl From for io::Error { + fn from(error: FsError) -> Self { + let kind = match error { + FsError::BadDescriptor | FsError::NoEntry => io::ErrorKind::NotFound, + FsError::NotPermitted => io::ErrorKind::PermissionDenied, + FsError::InvalidPath => io::ErrorKind::InvalidInput, + FsError::FileTooLarge | FsError::Overflow => io::ErrorKind::FileTooLarge, + FsError::QuotaExceeded => io::ErrorKind::QuotaExceeded, + FsError::Io(message) => return io::Error::other(message), + }; + io::Error::new(kind, error) + } +} + // --------------------------------------------------------------------------- // Filesystem limits // --------------------------------------------------------------------------- @@ -151,6 +169,52 @@ pub enum DescriptorType { pub struct DescriptorStat { pub descriptor_type: DescriptorType, pub size: u64, + /// Last access, modification and status change, where the host + /// keeps them. + pub accessed: Option, + pub modified: Option, + pub changed: Option, +} + +impl DescriptorStat { + fn of(metadata: &cap_std::fs::Metadata) -> Self { + let time = |t: io::Result| t.ok().map(|t| t.into_std()); + Self { + descriptor_type: if metadata.is_dir() { + DescriptorType::Directory + } else { + DescriptorType::RegularFile + }, + size: metadata.len(), + accessed: time(metadata.accessed()), + modified: time(metadata.modified()), + changed: changed(metadata), + } + } +} + +/// The status change time of `metadata`; hosts other than Unix keep none. +#[cfg(unix)] +fn changed(metadata: &cap_std::fs::Metadata) -> Option { + use std::time::{Duration, UNIX_EPOCH}; + + use cap_std::fs::MetadataExt; + + // Before the epoch, stat gives negative seconds and positive + // nanoseconds. + let (secs, nanos) = (metadata.ctime(), u32::try_from(metadata.ctime_nsec()).ok()?); + if secs >= 0 { + UNIX_EPOCH.checked_add(Duration::new(secs.unsigned_abs(), nanos)) + } else { + UNIX_EPOCH + .checked_sub(Duration::from_secs(secs.unsigned_abs()))? + .checked_add(Duration::from_nanos(nanos.into())) + } +} + +#[cfg(not(unix))] +fn changed(_: &cap_std::fs::Metadata) -> Option { + None } /// Effective permissions reported for a descriptor. @@ -303,6 +367,19 @@ fn handle_identity(handle: std::os::windows::io::RawHandle) -> Result Option { + use cap_std::fs::MetadataExt; + + Some((a.dev(), a.ino()) == (b.dev(), b.ino())) +} + +#[cfg(not(unix))] +fn same_file(_: &cap_std::fs::Metadata, _: &cap_std::fs::Metadata) -> Option { + None +} + // --------------------------------------------------------------------------- // Internal types // --------------------------------------------------------------------------- @@ -342,6 +419,10 @@ struct PreopenEntry { struct FilesystemUsage { logical_bytes: u64, file_count: usize, + // Entries of every kind, directories too, whatever the limits: what + // the output scan walks, kept by the path operations so a guest + // cannot make more than it can walk. + entries: usize, } /// Capability-based virtual filesystem. @@ -557,6 +638,9 @@ impl CapFs { None => anyhow::anyhow!("failed to write output file {key}: {error}"), }); } + if !existed { + self.output_usage.entries += 1; + } self.apply_mutation(old_size, new_size, !existed) .map_err(|error| anyhow::anyhow!(error))?; Ok(()) @@ -770,12 +854,18 @@ impl CapFs { return Ok(DescriptorStat { descriptor_type: DescriptorType::Directory, size: 0, + accessed: None, + modified: None, + changed: None, }); } if let Some(size) = self.file_size(fd) { return Ok(DescriptorStat { descriptor_type: DescriptorType::RegularFile, size, + accessed: None, + modified: None, + changed: None, }); } Err(FsError::BadDescriptor) @@ -788,15 +878,7 @@ impl CapFs { return Err(FsError::NotPermitted); } let metadata = dir.cap_std().metadata(path).map_err(|_| FsError::NoEntry)?; - let descriptor_type = if metadata.is_dir() { - DescriptorType::Directory - } else { - DescriptorType::RegularFile - }; - Ok(DescriptorStat { - descriptor_type, - size: metadata.len(), - }) + Ok(DescriptorStat::of(&metadata)) } /// Get the effective permission flags for a descriptor. @@ -1203,6 +1285,395 @@ impl CapFs { .collect() } + // ----------------------------------------------------------------------- + // Path operations + // ----------------------------------------------------------------------- + // + // For a guest with a filesystem of its own, which asks the host for + // each operation by path rather than through WASI descriptors (a + // hyperlight-unikraft guest's hostfs mounts). Paths are relative to + // the directory registered at `guest_root` ("/input", "/output"), + // `/`-separated, the empty path being that directory. The rules of the + // descriptor operations apply, before the host directory is touched: + // its permissions, and for the output the filesystem limits and no + // links. Errors are the ones the guest's file calls should see. + + /// Maximum entries in one directory listing, as for directory streams. + const MAX_LIST_ENTRIES: usize = Self::MAX_DIR_ENTRIES; + + /// The directory registered at `guest_root`, and whether it is the + /// output (the one the limits are kept for), once `path` is known to + /// stay in it as written. + fn path_dir(&self, guest_root: &str, path: &str) -> io::Result<(Dir, bool)> { + // cap-std refuses what escapes the directory; this refuses what + // only reads differently on another host (`\`, `:` on Windows) + // and what the output scan could not walk. + if !path.is_empty() { + let mut depth = 0usize; + for component in path.split('/') { + depth += 1; + if component.is_empty() + || component == "." + || component == ".." + || component.contains(['\\', '\0']) + || (cfg!(windows) && component.contains(':')) + || depth > Self::MAX_SCAN_DEPTH + { + return Err(FsError::InvalidPath.into()); + } + } + } + let (fd, entry) = self + .preopen_dirs + .iter() + .find(|(_, e)| e.guest_path == guest_root) + .ok_or(FsError::NoEntry)?; + Ok((entry.dir.clone(), self.output_fd == Some(*fd))) + } + + /// Not permitted unless `allowed`. + fn permit(allowed: bool) -> io::Result<()> { + if allowed { + Ok(()) + } else { + Err(FsError::NotPermitted.into()) + } + } + + /// Metadata of a path. + /// + /// Allowed when the directory may be read or changed: a guest's own + /// filesystem looks a path up before it creates or writes it, so a + /// write-only output needs this much. File contents still need + /// `FilePerms::READ`. + pub fn stat_path(&self, guest_root: &str, path: &str) -> io::Result { + let (dir, _) = self.path_dir(guest_root, path)?; + Self::permit(dir.perms().intersects(DirPerms::READ | DirPerms::MUTATE))?; + let metadata = if path.is_empty() { + dir.cap_std().dir_metadata()? + } else { + dir.cap_std().metadata(path)? + }; + Ok(DescriptorStat::of(&metadata)) + } + + /// Read into `buf` from `offset`; fewer bytes than `buf` holds means + /// the end of the file. + pub fn read_path( + &self, + guest_root: &str, + path: &str, + offset: u64, + buf: &mut [u8], + ) -> io::Result { + let (dir, _) = self.path_dir(guest_root, path)?; + Self::permit(dir.file_perms().contains(FilePerms::READ))?; + // Only regular files: opening a FIFO the host left would block the + // host call, with this `CapFs` locked. + if !dir.cap_std().metadata(path)?.is_file() { + return Err(FsError::NotPermitted.into()); + } + let mut file = dir.cap_std().open(path)?; + file.seek(SeekFrom::Start(offset))?; + let mut filled = 0; + while filled < buf.len() { + match file.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + Ok(filled) + } + + /// Write `data` at `offset`, or at the end for `None`, creating the + /// file when the directory may be changed. + pub fn write_path( + &mut self, + guest_root: &str, + path: &str, + offset: Option, + data: &[u8], + ) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, path)?; + Self::permit(dir.file_perms().contains(FilePerms::WRITE))?; + let len = u64::try_from(data.len()).map_err(|_| FsError::Overflow)?; + let mut opts = cap_std::fs::OpenOptions::new(); + opts.read(true).write(true); + let (mut file, old_size) = match dir.cap_std().open_with(path, &opts) { + Ok(file) => { + let metadata = file.metadata()?; + if output { + Self::refuse_links(&dir, path, metadata.clone())?; + } + (file, Some(metadata.len())) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + Self::permit(dir.perms().contains(DirPerms::MUTATE))?; + let end = offset + .unwrap_or(0) + .checked_add(len) + .ok_or(FsError::Overflow)?; + // Checked before the file exists, so a refused write + // leaves nothing behind. + if output { + self.reserve_entry()?; + self.validate_mutation(0, end, true)?; + } + opts.create_new(true); + let file = dir.cap_std().open_with(path, &opts)?; + if output { + self.output_usage.entries += 1; + } + (file, None) + } + Err(error) => return Err(error), + }; + let size = old_size.unwrap_or(0); + let at = offset.unwrap_or(size); + let new_size = size.max(at.checked_add(len).ok_or(FsError::Overflow)?); + if output && old_size.is_some() { + self.validate_mutation(size, new_size, false)?; + } + let written = file + .seek(SeekFrom::Start(at)) + .and_then(|_| file.write_all(data)); + if output { + match written { + Ok(()) => self.apply_mutation(size, new_size, old_size.is_none())?, + Err(_) => { + let _ = self.resynchronize_output(); + } + } + } + written + } + + /// Make a directory. + pub fn create_dir_path(&mut self, guest_root: &str, path: &str) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, path)?; + Self::permit(dir.perms().contains(DirPerms::MUTATE))?; + if output { + self.ensure_accounting_valid()?; + self.reserve_entry()?; + } + dir.cap_std().create_dir(path)?; + if output { + self.output_usage.entries += 1; + } + Ok(()) + } + + /// Remove a file or an empty directory. + pub fn remove_path(&mut self, guest_root: &str, path: &str) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, path)?; + Self::permit(dir.perms().contains(DirPerms::MUTATE))?; + let metadata = dir.cap_std().symlink_metadata(path)?; + if metadata.is_dir() { + dir.cap_std().remove_dir(path)?; + } else { + dir.cap_std().remove_file(path)?; + } + if output { + self.release_entry(&dir, path, &metadata); + } + Ok(()) + } + + /// Set a file's length. + pub fn truncate_path(&mut self, guest_root: &str, path: &str, len: u64) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, path)?; + Self::permit(dir.file_perms().contains(FilePerms::WRITE))?; + // Only regular files: opening a FIFO for writing would block. + if !dir.cap_std().metadata(path)?.is_file() { + return Err(FsError::NotPermitted.into()); + } + let file = dir + .cap_std() + .open_with(path, cap_std::fs::OpenOptions::new().write(true))?; + let metadata = file.metadata()?; + let size = metadata.len(); + if output { + Self::refuse_links(&dir, path, metadata)?; + self.validate_mutation(size, len, false)?; + } + let truncated = file.set_len(len); + if output { + match truncated { + Ok(()) => self.apply_mutation(size, len, false)?, + Err(_) => { + let _ = self.resynchronize_output(); + } + } + } + truncated + } + + /// Set the access and modification times, those that are `Some`: a + /// file's under `FilePerms::WRITE`, a directory's under + /// `DirPerms::MUTATE`. + pub fn set_times_path( + &self, + guest_root: &str, + path: &str, + accessed: Option, + modified: Option, + ) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, path)?; + let at = if path.is_empty() { "." } else { path }; + let metadata = dir.cap_std().metadata(at)?; + if metadata.is_dir() { + Self::permit(dir.perms().contains(DirPerms::MUTATE))?; + } else { + Self::permit(dir.file_perms().contains(FilePerms::WRITE))?; + // A hard link's times are those of a file outside output. + if output { + Self::refuse_links(&dir, path, metadata)?; + } + } + let spec = |t: Option| { + t.map(|t| SystemTimeSpec::Absolute(cap_std::time::SystemTime::from_std(t))) + }; + dir.cap_std().set_times(at, spec(accessed), spec(modified)) + } + + /// A directory's entries: each name, and whether it is a directory. + pub fn list_path(&self, guest_root: &str, path: &str) -> io::Result> { + let (dir, _) = self.path_dir(guest_root, path)?; + Self::permit(dir.perms().contains(DirPerms::READ))?; + let entries = if path.is_empty() { + dir.cap_std().entries()? + } else { + dir.cap_std().read_dir(path)? + }; + let mut listed = Vec::new(); + for entry in entries { + let entry = entry?; + if listed.len() >= Self::MAX_LIST_ENTRIES { + return Err(io::Error::other("directory listing exceeds entry limit")); + } + let name = entry.file_name(); + // As `stat_path` sees it: a link (the host's) by its target. + let is_dir = match entry.file_type() { + Ok(ft) if ft.is_symlink() => { + let at = Path::new(path).join(&name); + dir.cap_std().metadata(at).is_ok_and(|m| m.is_dir()) + } + ft => ft.is_ok_and(|ft| ft.is_dir()), + }; + listed.push((name.to_string_lossy().into_owned(), is_dir)); + } + Ok(listed) + } + + /// Rename `from` to `to`, replacing what `to` names as the host's + /// rename would. + pub fn rename_path(&mut self, guest_root: &str, from: &str, to: &str) -> io::Result<()> { + let (dir, output) = self.path_dir(guest_root, from)?; + self.path_dir(guest_root, to)?; + Self::permit(dir.perms().contains(DirPerms::MUTATE))?; + if from.is_empty() || to.is_empty() { + return Err(FsError::InvalidPath.into()); + } + if output { + self.ensure_accounting_valid()?; + } + let source = dir.cap_std().symlink_metadata(from)?; + // A directory moved deeper takes its own depth with it: what it + // holds must still be within what the output scan walks. + if source.is_dir() { + let depth = to.split('/').count(); + let room = Self::MAX_SCAN_DEPTH.saturating_sub(depth); + if !Self::height_within(&dir.cap_std().open_dir(from)?, room)? { + return Err(FsError::InvalidPath.into()); + } + } + let replaced = dir.cap_std().symlink_metadata(to).ok(); + dir.cap_std().rename(from, dir.cap_std(), to)?; + if output && let Some(replaced) = replaced { + match same_file(&source, &replaced) { + // Renamed onto itself: nothing changed. + Some(true) => {} + // Renamed over another, which is gone. + Some(false) => self.release_entry(&dir, to, &replaced), + // Not knowable here (a case-only rename, say): count again. + None => { + let _ = self.resynchronize_output(); + } + } + } + Ok(()) + } + + /// Whether nothing in `dir` is more than `room` levels below it, + /// walking no more than the output scan would. + fn height_within(dir: &CapDir, room: usize) -> io::Result { + Self::height_within_from(dir, room, &mut 0) + } + + // Depth first, so at most `room` (64) directories are open at a + // time, not every one found. + fn height_within_from(dir: &CapDir, room: usize, visited: &mut usize) -> io::Result { + for entry in dir.entries()? { + let entry = entry?; + *visited += 1; + if *visited > Self::MAX_SCAN_ENTRIES { + return Ok(false); + } + if entry.file_type()?.is_dir() + && (room == 0 + || !Self::height_within_from( + &dir.open_dir(entry.file_name())?, + room - 1, + visited, + )?) + { + return Ok(false); + } + } + Ok(true) + } + + /// Refuse an output file with another link, which may be outside it. + fn refuse_links(dir: &Dir, path: &str, metadata: cap_std::fs::Metadata) -> io::Result<()> { + Self::validated_output_metadata(dir.cap_std(), std::ffi::OsStr::new(path), metadata) + .map(drop) + .map_err(|error| io::Error::new(io::ErrorKind::PermissionDenied, error)) + } + + /// Refuse an output entry past what the output scan walks. + fn reserve_entry(&self) -> io::Result<()> { + if self.output_usage.entries >= Self::MAX_SCAN_ENTRIES { + return Err(FsError::QuotaExceeded.into()); + } + Ok(()) + } + + /// Account for the output entry `path` was, as `metadata` saw it, + /// now gone. + fn release_entry(&mut self, dir: &Dir, path: &str, metadata: &cap_std::fs::Metadata) { + self.output_usage.entries = self.output_usage.entries.saturating_sub(1); + if !metadata.is_file() || self.limits.is_unlimited() { + return; + } + // A file with another link keeps its bytes: count again. + let linked = Self::refuse_links(dir, path, metadata.clone()).is_err(); + match ( + self.output_usage.logical_bytes.checked_sub(metadata.len()), + self.output_usage.file_count.checked_sub(1), + ) { + (Some(bytes), Some(count)) if !linked => { + self.output_usage.logical_bytes = bytes; + self.output_usage.file_count = count; + } + // Or the accounting was off. + _ => { + let _ = self.resynchronize_output(); + } + } + } + // ----------------------------------------------------------------------- // Internals // ----------------------------------------------------------------------- @@ -1387,6 +1858,7 @@ impl CapFs { &mut usage, refuse_special, )?; + usage.entries = visited; Ok(usage) } @@ -2764,4 +3236,314 @@ mod tests { Err(FsError::Io("file descriptor handle space exhausted".into())) ); } + + // ----------------------------------------------------------------------- + // Path operations + // ----------------------------------------------------------------------- + + fn kind(result: io::Result) -> io::ErrorKind { + result.unwrap_err().kind() + } + + #[test] + fn paths_reach_nested_directories() { + let (mut fs, _input, output) = test_fs(); + fs.create_dir_path("/output", "a").unwrap(); + fs.create_dir_path("/output", "a/b").unwrap(); + fs.write_path("/output", "a/b/f.txt", None, b"hello") + .unwrap(); + fs.write_path("/output", "a/b/f.txt", None, b" world") + .unwrap(); + fs.write_path("/output", "a/b/f.txt", Some(0), b"J") + .unwrap(); + assert_eq!( + std::fs::read(output.path().join("a/b/f.txt")).unwrap(), + b"Jello world" + ); + + let mut buf = [0u8; 64]; + let n = fs.read_path("/output", "a/b/f.txt", 6, &mut buf).unwrap(); + assert_eq!(&buf[..n], b"world"); + let stat = fs.stat_path("/output", "a/b/f.txt").unwrap(); + assert_eq!(stat.descriptor_type, DescriptorType::RegularFile); + assert_eq!(stat.size, 11); + assert_eq!( + fs.stat_path("/output", "").unwrap().descriptor_type, + DescriptorType::Directory + ); + assert_eq!( + fs.list_path("/output", "a").unwrap(), + vec![("b".to_string(), true)] + ); + + fs.rename_path("/output", "a/b/f.txt", "g.txt").unwrap(); + assert_eq!( + kind(fs.remove_path("/output", "a")), + io::ErrorKind::DirectoryNotEmpty + ); + fs.remove_path("/output", "a/b").unwrap(); + fs.remove_path("/output", "a").unwrap(); + fs.truncate_path("/output", "g.txt", 2).unwrap(); + assert_eq!(std::fs::read(output.path().join("g.txt")).unwrap(), b"Je"); + } + + #[test] + fn paths_that_leave_or_misread_the_directory_are_refused() { + let (mut fs, _input, _output) = test_fs(); + for path in ["..", "a/../b", "./a", "a//b", "a/", "/a", "a\\b", "a\0b"] { + assert_eq!( + kind(fs.write_path("/output", path, None, b"x")), + io::ErrorKind::InvalidInput, + "{path:?}" + ); + } + let deep = vec!["d"; CapFs::MAX_SCAN_DEPTH + 1].join("/"); + assert_eq!( + kind(fs.create_dir_path("/output", &deep)), + io::ErrorKind::InvalidInput + ); + assert_eq!( + kind(fs.stat_path("/elsewhere", "")), + io::ErrorKind::NotFound + ); + assert_eq!( + kind(fs.rename_path("/output", "", "x")), + io::ErrorKind::InvalidInput + ); + } + + #[test] + fn input_paths_are_read_only() { + let (mut fs, input, _output) = test_fs(); + host_write_input(&input, "data.txt", b"in"); + let mut buf = [0u8; 8]; + assert_eq!(fs.read_path("/input", "data.txt", 0, &mut buf).unwrap(), 2); + assert_eq!( + kind(fs.write_path("/input", "data.txt", None, b"x")), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.write_path("/input", "new.txt", None, b"x")), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.create_dir_path("/input", "d")), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.remove_path("/input", "data.txt")), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.truncate_path("/input", "data.txt", 0)), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.rename_path("/input", "data.txt", "x")), + io::ErrorKind::PermissionDenied + ); + assert!(input.path().join("data.txt").exists()); + assert!(!input.path().join("new.txt").exists()); + } + + #[test] + fn path_writes_are_held_to_the_limits_before_they_happen() { + let (mut fs, output) = limited_output_fs(8, 12, 2); + assert_eq!( + kind(fs.write_path("/output", "big", None, &[0; 9])), + io::ErrorKind::FileTooLarge + ); + assert!( + !output.path().join("big").exists(), + "a refused write creates nothing" + ); + + fs.write_path("/output", "a", None, &[0; 8]).unwrap(); + assert_eq!( + kind(fs.write_path("/output", "a", Some(4), &[0; 5])), + io::ErrorKind::FileTooLarge + ); + // Rewriting in place grows nothing. + fs.write_path("/output", "a", Some(0), &[1; 8]).unwrap(); + fs.write_path("/output", "b", None, &[0; 4]).unwrap(); + assert_eq!( + kind(fs.write_path("/output", "b", None, &[0; 1])), + io::ErrorKind::QuotaExceeded + ); + assert_eq!( + kind(fs.truncate_path("/output", "b", 5)), + io::ErrorKind::QuotaExceeded + ); + assert_eq!( + kind(fs.write_path("/output", "c", None, b"")), + io::ErrorKind::QuotaExceeded + ); + assert_eq!(std::fs::metadata(output.path().join("b")).unwrap().len(), 4); + + // What is removed, truncated or renamed over is given back. + fs.truncate_path("/output", "a", 0).unwrap(); + fs.write_path("/output", "b", None, &[0; 4]).unwrap(); + fs.rename_path("/output", "b", "a").unwrap(); + fs.write_path("/output", "c", None, &[0; 4]).unwrap(); + fs.remove_path("/output", "c").unwrap(); + fs.write_path("/output", "d", None, &[0; 4]).unwrap(); + fs.validate_output().unwrap(); + assert_eq!(fs.output_usage.logical_bytes, 12); + assert_eq!(fs.output_usage.file_count, 2); + } + + #[cfg(unix)] + #[test] + fn path_writes_refuse_hard_linked_files() { + let (mut fs, output) = limited_output_fs(8, 12, 4); + fs.write_path("/output", "a", None, b"x").unwrap(); + std::fs::hard_link(output.path().join("a"), output.path().join("b")).unwrap(); + assert_eq!( + kind(fs.write_path("/output", "b", None, b"y")), + io::ErrorKind::PermissionDenied + ); + assert_eq!(std::fs::read(output.path().join("a")).unwrap(), b"x"); + } + + #[test] + fn directories_cannot_be_moved_past_the_scan_depth() { + let (mut fs, _input, _output) = test_fs(); + let chain = vec!["z"; 40].join("/"); + fs.create_dir_path("/output", "t").unwrap(); + fs.create_dir_path("/output", "t/u").unwrap(); + let mut at = String::new(); + for component in chain.split('/') { + at = if at.is_empty() { + component.to_string() + } else { + format!("{at}/{component}") + }; + fs.create_dir_path("/output", &at).unwrap(); + } + // `t` holds one level: moved under 63 levels, `t/u` would be 65 + // deep; under 62, it is 64. + let deep = vec!["z"; 63].join("/"); + for component in deep.split('/').skip(40) { + at = format!("{at}/{component}"); + fs.create_dir_path("/output", &at).unwrap(); + } + assert_eq!( + kind(fs.rename_path("/output", "t", &format!("{at}/t"))), + io::ErrorKind::InvalidInput + ); + let shallower = at.rsplit_once('/').unwrap().0; + fs.rename_path("/output", "t", &format!("{shallower}/t")) + .unwrap(); + fs.validate_output().unwrap(); + } + + #[test] + fn output_entries_stop_at_what_the_scan_walks() { + let (mut fs, _input, _output) = test_fs(); + fs.create_dir_path("/output", "d").unwrap(); + assert_eq!(fs.output_usage.entries, 1); + fs.output_usage.entries = CapFs::MAX_SCAN_ENTRIES; + assert_eq!( + kind(fs.create_dir_path("/output", "e")), + io::ErrorKind::QuotaExceeded + ); + assert_eq!( + kind(fs.write_path("/output", "f", None, b"x")), + io::ErrorKind::QuotaExceeded + ); + fs.remove_path("/output", "d").unwrap(); + fs.create_dir_path("/output", "e").unwrap(); + fs.validate_output().unwrap(); + assert_eq!(fs.output_usage.entries, 1); + } + + #[test] + fn write_only_output_can_be_written_not_read() { + let output = tempfile::tempdir().unwrap(); + let mut fs = CapFs::new() + .with_output_dir(output.path(), DirPerms::MUTATE, FilePerms::WRITE) + .unwrap(); + fs.stat_path("/output", "").unwrap(); + fs.write_path("/output", "f", None, b"x").unwrap(); + assert_eq!(fs.stat_path("/output", "f").unwrap().size, 1); + let mut buf = [0u8; 4]; + assert_eq!( + kind(fs.read_path("/output", "f", 0, &mut buf)), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + kind(fs.list_path("/output", "")), + io::ErrorKind::PermissionDenied + ); + } + + #[cfg(unix)] + #[test] + fn truncate_and_remove_respect_hard_links() { + let (mut fs, output) = limited_output_fs(8, 12, 4); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("precious"), b"keep").unwrap(); + std::fs::hard_link(outside.path().join("precious"), output.path().join("l")).unwrap(); + assert_eq!( + kind(fs.truncate_path("/output", "l", 0)), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + std::fs::read(outside.path().join("precious")).unwrap(), + b"keep" + ); + // Removing the link leaves the bytes it shared counted for what + // still has them, which is nothing in the output. + fs.remove_path("/output", "l").unwrap(); + assert_eq!(fs.output_usage.logical_bytes, 0); + assert_eq!(fs.output_usage.file_count, 0); + } + + #[cfg(unix)] + #[test] + fn special_files_are_not_read() { + let (fs, input, _output) = test_fs(); + let fifo = input.path().join("fifo"); + assert!( + std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success() + ); + let mut buf = [0u8; 4]; + assert_eq!( + kind(fs.read_path("/input", "fifo", 0, &mut buf)), + io::ErrorKind::PermissionDenied + ); + } + + #[test] + fn wide_directories_rename_with_few_handles_open() { + let (mut fs, _input, _output) = test_fs(); + fs.create_dir_path("/output", "w").unwrap(); + for i in 0..3000 { + fs.create_dir_path("/output", &format!("w/{i}")).unwrap(); + } + fs.rename_path("/output", "w", "w2").unwrap(); + fs.validate_output().unwrap(); + } + + #[cfg(unix)] + #[test] + fn special_files_are_not_truncated() { + let (mut fs, _input, output) = test_fs(); + let fifo = output.path().join("fifo"); + assert!( + std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success() + ); + assert_eq!( + kind(fs.truncate_path("/output", "fifo", 0)), + io::ErrorKind::PermissionDenied + ); + } } From 6092e44b0081f4f95efb54f0b334bcdf54955917 Mon Sep 17 00:00:00 2001 From: danbugs Date: Wed, 21 Oct 2026 21:23:44 +0000 Subject: [PATCH 10/10] Serve Unikraft guest mounts through CapFs The guest's /input and /output are now hostfs mounts whose operations CapFs serves, through hyperlight-unikraft's FsBackend, instead of host directories the guest writes directly and CapFs checks after the run. A write past the filesystem limits, a link, or a path deeper than the output scan walks now fails in the guest (EFBIG, EDQUOT, EACCES, EINVAL) before the host directory changes, as in the other backends. The check before and after each run stays, for what no single operation shows: what the host changes in /output during a run, and more than 100,000 entries. The guest no longer holds the directories by path, so the check that they were not replaced before boot goes, and with it CapFs::is_same_dir and CapFs::input_path, which only it used. Signed-off-by: danbugs --- README.md | 2 +- src/hyperlight_sandbox/src/cap_fs.rs | 118 ------- .../core/hyperlight_sandbox/__init__.py | 21 +- src/unikraft_sandbox/src/lib.rs | 234 ++++++++----- src/unikraft_sandbox/tests/python_image.rs | 331 +++++++++++++++--- src/unikraft_sandbox/tests/relative_output.rs | 9 +- 6 files changed, 445 insertions(+), 270 deletions(-) diff --git a/README.md b/README.md index 9484415..9eea663 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ How it differs from the other backends: - Only the `python`, `python-shell` and `agent` images are supported, and they must be the hyperlight-unikraft release the crate is built against (`IMAGE_RELEASE`). - Guest memory is one size, the image's tested default (1.5 GiB for `agent`); set it with `Unikraft::scratch_mb` (`scratch_size="2Gi"` in Python, `WithScratchSize("2Gi")` in .NET). Heap and stack sizes do not apply: the Python and .NET SDKs refuse them, and the Rust builder's are ignored. -- `/input` and `/output` are host directories mounted into the guest when it boots, so the guest writes `/output` directly. `/output` is read-write if its permissions allow creating and writing files, else read-only; it cannot be write-only. As with the other backends, files in `/output` persist from run to run, and restoring a snapshot clears them; files the guest has open are interpreter state, so they stay open across runs and through a restore. The filesystem limits are checked after each run: a run that exceeded them, or that left a symbolic or hard link, a special file, or a tree deeper than 64 levels or over 100,000 entries in `/output`, fails, and `/output` is cleared of every file and link (earlier runs' included) and of the directories made during the run; directories it had when the run started are kept, emptied. The bytes did reach the host disk first, so put the output directory on quota-controlled storage. The same check runs before each run, for what the host put in `/output` since: a run that would start on such things is refused, and they are left in place. The guest keeps the directories it booted with, which are the ones the filesystem limits are checked on: replacing either path on the host afterwards changes nothing for the sandbox. +- `/input` and `/output` are mounted in the guest's own filesystem, and `CapFs`, which serves the other backends' WASI file calls, serves each file operation the guest makes on them, by the same rules: the directory permissions, and for `/output` the filesystem limits, no symbolic or hard links, and at most 64 levels and 100,000 entries. An operation that would break them fails in the guest (`EFBIG`, `EDQUOT`, `EACCES`, `EINVAL`) before it touches the host directory. `/output` is read-write if its permissions allow creating and writing files (without read permission, files can be made and written but not read or listed), else read-only. As with the other backends, files in `/output` persist from run to run, and restoring a snapshot clears them; files the guest has open are interpreter state, so they stay open across runs and through a restore. `/output` is also checked as a whole before and after each run, for what the host put there meanwhile (a tool writing there, say). A run that ends past the limits fails, and `/output` is cleared of every file and link (earlier runs' included) and of the directories made during the run; directories it had when the run started are kept, emptied. A run that would start past them is refused, and what is there is left in place. Replacing either directory on the host after the build changes nothing for the sandbox, which keeps the one it opened. - `http_get` / `http_post` go through the host and the `allow_domain` allowlist, as in the Wasm guest. Guest sockets (`requests`, `urllib`) are off unless `Unikraft::native_network(true)` (`native_network=True`), and may then make TCP connections to an `allow_domain` entry's host on its port (the one the URL names, else its scheme's: 443 for `https`). The host checks every DNS question and connect against the allowlist as it is at that moment, so an `allow_domain` made later applies from the next connect. The host sees a connection's address and port, and recognises an allowed name's addresses through the answers DNS gives the guest, but not what goes over the connection: another site served from the same address (a CDN, a shared host) is reachable on that port too, and an entry's path and method filters apply to `http_get` / `http_post` only. An entry that allows no method opens no sockets. - One host call carries at most 64 KiB; larger code, tool arguments and results, and HTTP bodies are chunked transparently. The host holds at most 64 MiB of them unread at a time, so a single tool result or HTTP response can be up to that size. - Exit statuses are those a CPython process exits with, 0 to 255 (`sys.exit(-1)` is 255, `sys.exit(256)` is 0), so -1 always means the guest itself failed. diff --git a/src/hyperlight_sandbox/src/cap_fs.rs b/src/hyperlight_sandbox/src/cap_fs.rs index cd075bc..13b6305 100644 --- a/src/hyperlight_sandbox/src/cap_fs.rs +++ b/src/hyperlight_sandbox/src/cap_fs.rs @@ -292,81 +292,6 @@ impl Dir { } } -/// A directory's identity: device and inode on Unix, volume serial number -/// and 128-bit file id on Windows. -#[cfg(unix)] -type DirIdentity = (u64, u64); -#[cfg(windows)] -type DirIdentity = (u64, [u8; 16]); - -#[cfg(unix)] -fn dir_identity(dir: &CapDir) -> Result { - use cap_std::fs::MetadataExt; - - let meta = dir - .dir_metadata() - .context("failed to read directory metadata")?; - Ok((meta.dev(), meta.ino())) -} - -#[cfg(unix)] -fn path_identity(path: &Path) -> Result { - use std::os::unix::fs::MetadataExt; - - let meta = std::fs::metadata(path) - .with_context(|| format!("failed to read metadata of {}", path.display()))?; - Ok((meta.dev(), meta.ino())) -} - -#[cfg(windows)] -fn dir_identity(dir: &CapDir) -> Result { - use std::os::windows::io::AsRawHandle; - - handle_identity(dir.as_raw_handle()) -} - -#[cfg(windows)] -fn path_identity(path: &Path) -> Result { - use std::os::windows::fs::OpenOptionsExt; - use std::os::windows::io::AsRawHandle; - - use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS; - - // Backup semantics are what lets a directory be opened as a file. - let file = std::fs::OpenOptions::new() - .read(true) - .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) - .open(path) - .with_context(|| format!("failed to open {}", path.display()))?; - handle_identity(file.as_raw_handle()) -} - -/// The 128-bit file id, unlike `GetFileInformationByHandle`'s 64-bit index, -/// is unique on ReFS too. -#[cfg(windows)] -fn handle_identity(handle: std::os::windows::io::RawHandle) -> Result { - use std::mem::MaybeUninit; - - use windows_sys::Win32::Storage::FileSystem::{ - FILE_ID_INFO, FileIdInfo, GetFileInformationByHandleEx, - }; - - let mut info = MaybeUninit::::zeroed(); - let ok = unsafe { - GetFileInformationByHandleEx( - handle as _, - FileIdInfo, - info.as_mut_ptr().cast(), - std::mem::size_of::() as u32, - ) - }; - if ok == 0 { - return Err(std::io::Error::last_os_error()).context("failed to read the file id"); - } - let info = unsafe { info.assume_init() }; - Ok((info.VolumeSerialNumber, info.FileId.Identifier)) -} - /// Whether two metadata are of the same file, where that can be told. #[cfg(unix)] fn same_file(a: &cap_std::fs::Metadata, b: &cap_std::fs::Metadata) -> Option { @@ -439,8 +364,6 @@ pub struct CapFs { dir_streams: HashMap, next_handle: u32, - // Host filesystem path to the input directory. - input_path: Option, // Host filesystem path to the output directory. output_path: Option, // Owns the output temp dir when using the default. @@ -475,7 +398,6 @@ impl CapFs { streams: HashMap::new(), dir_streams: HashMap::new(), next_handle: FIRST_PREOPEN_FD, - input_path: None, output_path: None, _output_tmp: None, limits, @@ -515,7 +437,6 @@ impl CapFs { guest_path: "/input".to_string(), }, ); - self.input_path = Some(input_path.as_ref().to_path_buf()); Ok(self) } @@ -687,14 +608,6 @@ impl CapFs { Ok(data) } - /// Return the host filesystem path of the input directory, if configured. - /// - /// Backends that hand the guest a host directory directly (rather than - /// serving it through this `CapFs`) need the path to mount it. - pub fn input_path(&self) -> Option<&Path> { - self.input_path.as_deref() - } - /// Return the host filesystem path of the output directory, if configured. pub fn output_path(&self) -> Option<&Path> { self.output_path.as_deref() @@ -784,19 +697,6 @@ impl CapFs { self.preopen_dirs.contains_key(&fd) } - /// Whether `path` names the directory opened for `guest_path` (the same - /// file, not merely the same name): false once it was replaced, say. - /// - /// Backends that hand the guest a host directory by path, as a mount, - /// check it before doing so, since this `CapFs` checks and clears the - /// directory it opened. - pub fn is_same_dir(&self, guest_path: &str, path: &Path) -> Result { - let dir = self - .dir_by_guest_path(guest_path) - .ok_or_else(|| anyhow::anyhow!("no directory is open for {guest_path}"))?; - Ok(dir_identity(dir.cap_std())? == path_identity(path)?) - } - /// Find a preopened directory by its guest-visible path. pub fn dir_by_guest_path(&self, guest_path: &str) -> Option<&Dir> { self.preopen_dirs @@ -2481,24 +2381,6 @@ mod tests { ); } - #[test] - fn is_same_dir_notices_a_replaced_directory() { - let (fs, output) = limited_output_fs(8, 32, 4); - assert!(fs.is_same_dir("/output", output.path()).unwrap()); - let moved = output.path().with_extension("moved"); - match std::fs::rename(output.path(), &moved) { - Ok(()) => {} - // Windows does not rename a directory this `CapFs` holds open, - // so it cannot be replaced at all there. - Err(_) if cfg!(windows) => return, - Err(error) => panic!("{error}"), - } - std::fs::create_dir(output.path()).unwrap(); - assert!(!fs.is_same_dir("/output", output.path()).unwrap()); - assert!(fs.is_same_dir("/output", &moved).unwrap()); - std::fs::remove_dir_all(&moved).unwrap(); - } - #[test] fn validate_output_checks_what_the_guest_wrote_directly() { let (mut fs, output) = limited_output_fs(8, 32, 4); diff --git a/src/sdk/python/core/hyperlight_sandbox/__init__.py b/src/sdk/python/core/hyperlight_sandbox/__init__.py index ea9e77a..60c043b 100644 --- a/src/sdk/python/core/hyperlight_sandbox/__init__.py +++ b/src/sdk/python/core/hyperlight_sandbox/__init__.py @@ -135,15 +135,18 @@ class Sandbox: overrides the guest memory the image was tested with. ``native_network`` also lets guest sockets (``requests``, ``urllib``) reach the domains passed to ``allow_domain`` over TCP, on each entry's port (path and method - filters apply to ``http_get`` / ``http_post`` only). ``/input`` and - ``/output`` are host directories mounted at boot. As with the other - backends, files in ``/output`` persist from run to run and ``restore()`` - clears them; open guest files are interpreter state, and stay open too. - A run that leaves a symbolic or hard link, a special file or files past - the filesystem limits in ``/output`` fails, and ``/output`` is cleared of - every file and link (earlier runs' included) and of the directories made - during the run; those it had when the run started are kept. A run is - refused if ``/output`` holds such things when it would start. Exit + filters apply to ``http_get`` / ``http_post`` only). The guest's file + operations on ``/input`` and ``/output`` are each checked against the + directory permissions and filesystem limits, as the other backends' + are, and one that would break them fails in the guest (``OSError``) + before it reaches the host directory; links cannot be made. As with the + other backends, files in ``/output`` persist from run to run and + ``restore()`` clears them; open guest files are interpreter state, and + stay open too. ``/output`` is also checked as a whole before and after + each run, for what the host changed meanwhile: a run that ends past the limits fails, and ``/output`` is + cleared of every file and link (earlier runs' included) and of the + directories made during the run; a run is refused if ``/output`` is + past them when it would start. Exit statuses are a CPython process's, 0 to 255 (``sys.exit(-1)`` is 255), so -1 means the guest itself failed. Tool arguments and results and HTTP bodies of any size are exchanged in diff --git a/src/unikraft_sandbox/src/lib.rs b/src/unikraft_sandbox/src/lib.rs index e406b83..aab3e58 100644 --- a/src/unikraft_sandbox/src/lib.rs +++ b/src/unikraft_sandbox/src/lib.rs @@ -11,7 +11,7 @@ //! | `run(code)` | a guest function call into a boot-time bridge | //! | `call_tool(name, **kwargs)` | host function `sandbox.call_tool` | //! | `http_get` / `http_post` | host function `sandbox.http` (allowlisted) | -//! | `/input`, `/output` | hostfs mounts (read-only / read-write) | +//! | `/input`, `/output` | hostfs mounts served by `CapFs` | //! | `allow_domain` | checked per HTTP request and, opt-in via [`Unikraft::native_network`], per socket connect | //! | `snapshot` / `restore` | hyperlight snapshots of the whole micro-VM | //! @@ -25,19 +25,20 @@ use std::collections::{HashMap, HashSet}; use std::net::IpAddr; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{Context, Result}; use base64::Engine as _; use cap_std::fs::Dir as CapDir; use hyperlight_sandbox::runtime::BlockOn; use hyperlight_sandbox::{ - CapFs, DirPerms, ExecutionResult, FilePerms, Guest, GuestSandbox, HttpMethod, MethodFilter, - NetworkPermissions, SandboxConfig, Snapshot, ToolRegistry, http as sandbox_http, + CapFs, DescriptorType, DirPerms, ExecutionResult, FilePerms, Guest, GuestSandbox, HttpMethod, + MethodFilter, NetworkPermissions, SandboxConfig, Snapshot, ToolRegistry, http as sandbox_http, }; pub use hyperlight_unikraft; use hyperlight_unikraft::{ - AppSandbox, Destination, Filter, Mount, NetworkFilter, NetworkPolicy, - SandboxBuilder as HlukBuilder, Snapshot as HlukSnapshot, + AppSandbox, Destination, DirEntry, FileStat, Filter, FsBackend, Mount, NetworkFilter, + NetworkPolicy, SandboxBuilder as HlukBuilder, Snapshot as HlukSnapshot, }; /// The guest code that defines `call_tool`, `http_get`, `http_post` and the @@ -118,22 +119,27 @@ pub fn cached_rootfs(runtime: &str) -> Result { /// with (1.5 GiB for `agent`, 256 MiB for `python`) and is set with /// [`scratch_mb`](Self::scratch_mb). /// -/// `/input` and `/output` are host directories mounted at boot. `/output` -/// is read-write when its permissions allow creating and writing files, -/// else read-only when they allow reading, else absent; it cannot be -/// write-only. As in the other backends, its files persist from run to +/// `/input` and `/output` are mounted in the guest's filesystem, and each +/// file operation the guest makes on them is served by `CapFs`, by the +/// rules the other backends' WASI calls get: its permissions, and for +/// `/output` the filesystem limits, no links, and at most 64 levels and +/// 100,000 entries. An operation that would break them fails in the guest +/// (`EFBIG`, `EDQUOT`, `EACCES`, `EINVAL`), before the host directory is +/// touched. `/output` is read-write when its permissions allow creating +/// and writing files (write-only when they do not allow reading: files +/// can be made and written, not read or listed), else read-only when they +/// allow reading, else absent. As in the other backends, its files persist from run to /// run, and restoring a snapshot clears them. Files the guest has open are /// interpreter state, though: they stay open across runs, and a restore -/// does not close them. Since the guest writes `/output` directly, it is -/// checked by the rules `CapFs` applies to writes made through it (the -/// filesystem limits, no symbolic or hard links, no special files, at most -/// 64 levels and 100,000 entries) after each run, and before each, as the -/// host may have changed it since. A run that breaks them fails, and -/// `/output` is cleared of every file and link and of the directories made -/// during the run; those it had when the run started are kept, emptied. A -/// run that would start on such things is refused, and they are left as -/// they are. The guest is booted only onto -/// the directories the sandbox was built with. +/// does not close them. +/// +/// `/output` is also checked as a whole after each run, and before each, +/// for what the host may have changed in it meanwhile: the limits, no +/// links or special files, and the 64 levels and 100,000 entries. A run +/// that ends past them fails, and `/output` is cleared of every file and +/// link and of the directories made during the run; those it had when the +/// run started are kept, emptied. A run that would start past them is +/// refused, and what is there is left as it is. /// /// A run's exit status is the one a CPython process would exit with, 0 to /// 255 (`sys.exit(-1)` is 255), so -1 always means the guest failed. @@ -491,7 +497,6 @@ impl UnikraftGuestSandbox { if let Some(mb) = guest.scratch_mb { builder = builder.scratch_mb(mb); } - check_mounts(&files, &mounts)?; let builder = configure(builder, &host, &mounts, policy); let mut sandbox = builder .boot() @@ -527,11 +532,11 @@ impl UnikraftGuestSandbox { .lock() .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; // What persists from the last run, and what the host has put - // there since, by the rules a run's output is checked by: the - // guest reaches `/output` itself, through its mount, so it must - // not start on a link or a special file. The rest of - // `prepare_for_run` is resetting `CapFs` handles, of which the - // guest holds none. + // there since, by the rules a run's output is checked by: + // `CapFs` checks each operation the guest makes, but not what + // it starts on, and resynchronises its accounting here. The + // rest of `prepare_for_run` is resetting `CapFs` handles, of + // which the guest holds none. files .validate_output() .context("refusing to run on what /output holds")?; @@ -598,13 +603,13 @@ impl UnikraftGuestSandbox { }) } - /// The guest writes `/output` through hostfs, which `CapFs` does not - /// see, so `CapFs` checks it once the run is over, by the rules it - /// applies to writes made through it: the filesystem limits, no - /// symbolic or hard links, no special files, and its traversal bounds - /// (64 levels, 100,000 entries). A run that breaks them fails and - /// `/output` is cleared, earlier runs' files with it: what the guest - /// broke them with cannot be told from the rest. + /// `CapFs` checks each operation the guest makes on `/output`, but + /// not what the host does to it during the run (a tool writing there, + /// say), so it checks the whole once the run is over: the filesystem limits, no symbolic or + /// hard links, no special files, and its traversal bounds (64 levels, + /// 100,000 entries). A run that breaks them fails and `/output` is + /// cleared, earlier runs' files with it: what broke them cannot be + /// told from the rest. /// /// Returns what to say in the run's stderr, if anything. fn check_output_limits(&self) -> Option { @@ -732,32 +737,136 @@ fn strip_stderr_lines(output: &str, stderr: &str) -> Option { /// `/input` read-only, and `/output` as close to its `CapFs` permissions as /// a hostfs mount, read-only or read-write, can be: read-write when files /// may be created and written, else read-only when it may be read, else -/// not mounted. Write-only cannot be had: the guest can read back what it -/// wrote. -fn mounts_for(files: &Mutex) -> Result> { - let files = files +/// not mounted. Either way `CapFs` serves them, so its permissions and +/// limits decide every operation the guest makes: a mount without read +/// permission still refuses reads. +fn mounts_for(files: &Arc>) -> Result> { + let guarded = files .lock() .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; - // Resolved now, as `CapFs` resolved them: a relative path would - // follow the process's working directory, which may change. - let absolute = |path: &Path| { - std::fs::canonicalize(path).with_context(|| format!("cannot resolve {}", path.display())) + let mount = |root: &'static str, readonly| { + let backend = Arc::new(CapFsMount { + files: files.clone(), + root, + }); + Mount::backend(backend, root, readonly) }; let mut mounts = Vec::new(); - if let Some(input) = files.input_path() { - mounts.push(Mount::ro(absolute(input)?, "/input")); + if guarded.dir_by_guest_path("/input").is_some() { + mounts.push(mount("/input", true)); } - if let (Some(output), Some(dir)) = (files.output_path(), files.dir_by_guest_path("/output")) { - let output = absolute(output)?; + if let Some(dir) = guarded.dir_by_guest_path("/output") { if dir.perms().contains(DirPerms::MUTATE) && dir.file_perms().contains(FilePerms::WRITE) { - mounts.push(Mount::rw(output, "/output")); + mounts.push(mount("/output", false)); } else if dir.perms().contains(DirPerms::READ) { - mounts.push(Mount::ro(output, "/output")); + mounts.push(mount("/output", true)); } } Ok(mounts) } +/// A guest mount served by `CapFs`, by the path operations it has for a +/// guest with a filesystem of its own. +struct CapFsMount { + files: Arc>, + /// The directory's guest path, which `CapFs` knows it by. + root: &'static str, +} + +impl CapFsMount { + fn with( + &self, + op: impl FnOnce(&mut CapFs, &str) -> std::io::Result, + ) -> std::io::Result { + let mut files = self + .files + .lock() + .map_err(|_| std::io::Error::other("filesystem mutex poisoned"))?; + op(&mut files, self.root) + } +} + +impl FsBackend for CapFsMount { + fn stat(&self, path: &str) -> std::io::Result { + self.with(|files, root| { + let stat = files.stat_path(root, path)?; + // Permission bits say what `CapFs` would allow, so tools that + // check them first (`os.access`, `shutil.copy`) agree with it. + let (dir, file) = files + .dir_by_guest_path(root) + .map_or((DirPerms::empty(), FilePerms::empty()), |d| { + (d.perms(), d.file_perms()) + }); + let bits = |read: bool, write: bool| { + (if read { 0o444 } else { 0 }) | (if write { 0o200 } else { 0 }) + }; + let node = match stat.descriptor_type { + DescriptorType::Directory => FileStat::dir( + 0o111 | bits(dir.contains(DirPerms::READ), dir.contains(DirPerms::MUTATE)), + ), + DescriptorType::RegularFile => FileStat::file( + stat.size, + bits( + file.contains(FilePerms::READ), + file.contains(FilePerms::WRITE), + ), + ), + }; + // A host that keeps no status change time: the modification + // time stands in, as on hluk's own mounts. + let modified = stat.modified.unwrap_or(UNIX_EPOCH); + Ok(node.times( + stat.accessed.unwrap_or(UNIX_EPOCH), + modified, + stat.changed.unwrap_or(modified), + )) + }) + } + + fn read(&self, path: &str, offset: u64, buf: &mut [u8]) -> std::io::Result { + self.with(|files, root| files.read_path(root, path, offset, buf)) + } + + fn write(&self, path: &str, offset: Option, data: &[u8]) -> std::io::Result<()> { + self.with(|files, root| files.write_path(root, path, offset, data)) + } + + fn create_dir(&self, path: &str) -> std::io::Result<()> { + self.with(|files, root| files.create_dir_path(root, path)) + } + + fn remove(&self, path: &str) -> std::io::Result<()> { + self.with(|files, root| files.remove_path(root, path)) + } + + fn truncate(&self, path: &str, len: u64) -> std::io::Result<()> { + self.with(|files, root| files.truncate_path(root, path, len)) + } + + fn list(&self, path: &str) -> std::io::Result> { + self.with(|files, root| { + Ok(files + .list_path(root, path)? + .into_iter() + .map(|(name, is_dir)| DirEntry::new(name, is_dir)) + .collect()) + }) + } + + fn rename(&self, from: &str, to: &str) -> std::io::Result<()> { + self.with(|files, root| files.rename_path(root, from, to)) + } + + fn set_times( + &self, + path: &str, + accessed: Option, + modified: Option, + ) -> std::io::Result<()> { + self.with(|files, root| files.set_times_path(root, path, accessed, modified)) + } +} + /// What guest sockets may reach: TCP connections to the host of an /// `allow_domain` entry on its port, looked up in the allowlist as it is /// at each DNS question and connect. A name is recognised by address, @@ -844,39 +953,6 @@ impl NetworkFilter for SocketFilter { } } -/// Refuse to mount a directory other than the one `CapFs` opened for it. -/// -/// hostfs opens each mount by path when the guest boots and keeps that -/// directory from then on (a restore keeps it too), while `CapFs` checks -/// and clears the one it opened at build. So this is checked before the -/// boot: were the path replaced since the build (`rm -rf out && mkdir -/// out`), the guest would write a directory nothing checks. After the -/// boot, replacing it changes nothing for either side. -fn check_mounts(files: &Mutex, mounts: &[Mount]) -> Result<()> { - let files = files - .lock() - .map_err(|_| anyhow::anyhow!("filesystem mutex poisoned"))?; - for mount in mounts { - let same = files - .is_same_dir(&mount.guest_path, &mount.host_path) - .with_context(|| { - format!( - "cannot check that {} ({}) is the directory the sandbox was built with", - mount.guest_path, - mount.host_path.display() - ) - })?; - anyhow::ensure!( - same, - "{} ({}) was replaced since the sandbox was built; restore the original \ - directory or build a new sandbox", - mount.guest_path, - mount.host_path.display() - ); - } - Ok(()) -} - fn configure( builder: HlukBuilder, host: &HostState, diff --git a/src/unikraft_sandbox/tests/python_image.rs b/src/unikraft_sandbox/tests/python_image.rs index bc1066e..1a4b963 100644 --- a/src/unikraft_sandbox/tests/python_image.rs +++ b/src/unikraft_sandbox/tests/python_image.rs @@ -207,8 +207,11 @@ fn guest_classes_pickle_as_in_a_python_process() { assert_eq!(r.stdout, "3 __main__\nmain\n", "{}", r.stderr); } +/// The guest's file calls are refused the write that would break the +/// filesystem limits, as the other backends refuse it, before anything +/// reaches the host directory. #[test] -fn output_limits_are_enforced_after_the_run() { +fn output_limits_refuse_the_write_that_breaks_them() { let Some(rootfs) = rootfs("python") else { return; }; @@ -221,15 +224,25 @@ fn output_limits_are_enforced_after_the_run() { .build() .unwrap(); let r = sandbox - .run("open('/output/big.bin', 'wb').write(b'0' * 2048)") + .run( + "import errno, os\n\ + def attempt(path, size):\n \ + try:\n \ + with open(path, 'wb') as f:\n \ + f.write(b'0' * size)\n \ + return 'ok'\n \ + except OSError as e:\n \ + return errno.errorcode[e.errno]\n\ + print(attempt('/output/big.bin', 2048), os.path.getsize('/output/big.bin'))\n\ + os.remove('/output/big.bin')\n\ + print(attempt('/output/a', 1000), attempt('/output/b', 1000), attempt('/output/c', 1))", + ) .unwrap(); - assert_eq!(r.exit_code, 1); - assert!( - r.stderr.contains("exceeded the filesystem limits"), - "{}", - r.stderr - ); - assert!(sandbox.get_output_files().unwrap().is_empty()); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "EFBIG 0\nok ok EDQUOT\n", "{}", r.stderr); + let mut files = sandbox.get_output_files().unwrap(); + files.sort(); + assert_eq!(files, ["a", "b"]); } #[test] @@ -340,24 +353,36 @@ fn unread_replies_are_capped_per_run() { assert_eq!(r.stdout, "1048576\n", "{}", r.stderr); } +/// What the host puts in `/output` during a run, which the guest's file +/// calls could not have been refused, is checked once the run is over: +/// whatever the run's status, its output is kept and the check made. #[test] fn out_of_range_exit_status_keeps_output_and_limits() { let Some(rootfs) = rootfs("python") else { return; }; + let output = tempfile::tempdir().unwrap(); + let host_side = output.path().to_owned(); + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("spoil", move |_| { + std::fs::write(host_side.join("big.bin"), vec![0u8; 2048]).unwrap(); + Ok(serde_json::Value::Null) + }); let limits = hyperlight_sandbox::FilesystemLimits::new(1024, 4096, 2).unwrap(); let mut sandbox = SandboxBuilder::new() .module_path(rootfs) + .with_tools(tools) .filesystem_limits(limits) - .temp_output() + .output_dir( + output.path(), + hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::READ | hyperlight_sandbox::FilePerms::WRITE, + ) .guest(Unikraft::new()) .build() .unwrap(); let r = sandbox - .run( - "open('/output/big.bin', 'wb').write(b'0' * 2048)\n\ - print('before')\nimport sys\nsys.exit(2 ** 40)", - ) + .run("call_tool('spoil')\nprint('before')\nimport sys\nsys.exit(2 ** 40)") .unwrap(); assert_eq!(r.exit_code, 1); assert_eq!(r.stdout, "before\n"); @@ -369,22 +394,194 @@ fn out_of_range_exit_status_keeps_output_and_limits() { assert!(sandbox.get_output_files().unwrap().is_empty()); } +/// Through `CapFs`, `/input` and `/output` still work as a filesystem +/// does for the libraries a script uses. (Not `shutil.rmtree` or +/// `shutil.copytree`, which no hostfs mount supports yet: the guest gives +/// a path different inode numbers by `lstat` and `fstat`, and has no +/// `utime`.) +#[test] +fn input_and_output_work_as_a_filesystem() { + let input = tempfile::tempdir().unwrap(); + std::fs::create_dir(input.path().join("data")).unwrap(); + std::fs::write(input.path().join("data/in.txt"), "in").unwrap(); + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), Some(input.path())) else { + return; + }; + let r = sandbox + .run( + "import os, shutil, tempfile\n\ + print(open('/input/data/in.txt').read(), os.listdir('/input'))\n\ + os.makedirs('/output/a/b', exist_ok=True)\n\ + os.makedirs('/output/a/b', exist_ok=True)\n\ + with tempfile.NamedTemporaryFile('w', dir='/output/a', delete=False) as f:\n \ + f.write('atomic')\n\ + os.replace(f.name, '/output/a/b/result.txt')\n\ + with open('/output/log', 'a') as f:\n \ + f.write('one ')\n\ + with open('/output/log', 'a') as f:\n \ + f.write('two')\n\ + with open('/output/log', 'r+') as f:\n \ + f.seek(4)\n \ + f.write('TWO')\n \ + f.truncate()\n\ + os.makedirs('/output/copy/b')\n\ + shutil.copyfile('/output/a/b/result.txt', '/output/copy/b/result.txt')\n\ + for d, dirs, fs in os.walk('/output/a', topdown=False):\n \ + for name in fs:\n \ + os.remove(os.path.join(d, name))\n \ + os.rmdir(d)\n\ + print(sorted((d, sorted(fs)) for d, _, fs in os.walk('/output')))\n\ + try:\n \ + open('/input/new', 'w')\n\ + except OSError as e:\n \ + print(type(e).__name__)", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, + "in ['data']\n\ + [('/output', ['log']), ('/output/copy', []), ('/output/copy/b', ['result.txt'])]\n\ + OSError\n", + "{}", + r.stderr + ); + let output = sandbox.output_path().unwrap().unwrap(); + assert_eq!( + std::fs::read_to_string(output.join("log")).unwrap(), + "one TWO" + ); + assert_eq!( + std::fs::read_to_string(output.join("copy/b/result.txt")).unwrap(), + "atomic" + ); +} + +/// A directory cannot be moved where what it holds would be deeper than +/// the 64 levels `/output` may have. #[test] -fn symlinks_in_output_are_removed() { +fn directories_cannot_be_moved_too_deep() { let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { return; }; + let r = sandbox + .run( + "import errno, os\n\ + deep = '/output/' + '/'.join(['z'] * 63)\n\ + os.makedirs(deep)\n\ + os.makedirs('/output/t/u')\n\ + try:\n \ + os.rename('/output/t', deep + '/t')\n\ + except OSError as e:\n \ + print(errno.errorcode[e.errno])\n\ + os.rename('/output/t', os.path.dirname(deep) + '/t')\n\ + print('moved')", + ) + .unwrap(); + assert_eq!( + (r.exit_code, r.stdout.as_str()), + (0, "EINVAL\nmoved\n"), + "{}", + r.stderr + ); +} + +/// Without read permission `/output` is written, not read. +#[test] +fn write_only_output_is_written_not_read() { + let Some(rootfs) = rootfs("python") else { + return; + }; + let output = tempfile::tempdir().unwrap(); + let mut sandbox = SandboxBuilder::new() + .module_path(rootfs) + .output_dir( + output.path(), + hyperlight_sandbox::DirPerms::MUTATE, + hyperlight_sandbox::FilePerms::WRITE, + ) + .guest(Unikraft::new()) + .build() + .unwrap(); let r = sandbox .run( "import os\n\ - os.symlink('../../../../etc/passwd', '/output/escape')\n\ - open('/output/ok.txt', 'w').write('ok')", + open('/output/f', 'w').write('x')\n\ + print(os.access('/output/f', os.R_OK), os.access('/output/f', os.W_OK))\n\ + for op in (lambda: open('/output/f').read(), lambda: os.listdir('/output')):\n \ + try:\n \ + op()\n \ + print('read')\n \ + except PermissionError:\n \ + print('refused')", ) .unwrap(); - assert_eq!(r.exit_code, 1, "{}", r.stderr); - assert!(r.stderr.contains("symbolic links"), "{}", r.stderr); + assert_eq!(r.stdout, "False True\nrefused\nrefused\n", "{}", r.stderr); + assert_eq!( + std::fs::read_to_string(output.path().join("f")).unwrap(), + "x" + ); +} + +#[test] +fn links_cannot_be_made_in_output() { + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { + return; + }; + let r = sandbox + .run( + "import os\n\ + open('/output/a', 'w').write('a')\n\ + for make in (lambda: os.symlink('../../../../etc/passwd', '/output/escape'),\n \ + lambda: os.link('/output/a', '/output/b')):\n \ + try:\n \ + make()\n \ + print('made')\n \ + except PermissionError:\n \ + print('refused')", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(r.stdout, "refused\nrefused\n", "{}", r.stderr); + assert_eq!(sandbox.get_output_files().unwrap(), ["a"]); +} + +#[test] +fn file_times_reach_the_host() { + let input = tempfile::tempdir().unwrap(); + std::fs::write(input.path().join("in"), "in").unwrap(); + let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), Some(input.path())) else { + return; + }; + let r = sandbox + .run( + "import os\n\ + open('/output/a', 'w').write('a')\n\ + os.utime('/output/a', (1000000000, 1234567890))\n\ + s = os.stat('/output/a')\n\ + print(int(s.st_atime), int(s.st_mtime))\n\ + os.mkdir('/output/d')\n\ + os.utime('/output/d', (0, 1500000000))\n\ + print(int(os.stat('/output/d').st_mtime))\n\ + print(os.stat('/input/in').st_mtime > 0)\n\ + try:\n os.utime('/input/in', (0, 0))\nexcept OSError:\n print('refused')", + ) + .unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!( + r.stdout, "1000000000 1234567890\n1500000000\nTrue\nrefused\n", + "{}", + r.stderr + ); let output = sandbox.output_path().unwrap().unwrap(); - assert_eq!(std::fs::read_dir(output).unwrap().count(), 0); + let modified = std::fs::metadata(output.join("a")) + .unwrap() + .modified() + .unwrap(); + assert_eq!( + modified, + std::time::UNIX_EPOCH + std::time::Duration::from_secs(1234567890) + ); } /// This host's address on the interface it reaches the internet by: not @@ -482,39 +679,54 @@ fn native_network_follows_the_allowlist_without_a_reboot() { } #[test] -fn output_trees_past_the_walk_limits_fail_and_are_removed() { +fn output_trees_stop_at_the_walk_limits() { let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { return; }; let output = sandbox.output_path().unwrap().unwrap(); - for code in [ - // Deeper than the 64 levels CapFs walks. - "import os\nos.makedirs('/output/' + '/'.join(['d'] * 70))", - // More entries than the 100,000 it walks, directories counting. - "import os\nfor i in range(100100):\n os.mkdir(f'/output/{i}')", - ] { - let r = sandbox.run(code).unwrap(); - assert_eq!(r.exit_code, 1, "{}", r.stderr); - assert!(r.stderr.contains("its files were removed"), "{}", r.stderr); - assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); - // The next run is unaffected. - let r = sandbox.run("open('/output/ok', 'w').write('ok')").unwrap(); - assert_eq!(r.exit_code, 0, "{}", r.stderr); - assert_eq!(sandbox.get_output_files().unwrap(), vec!["ok".to_string()]); - } -} + // Deeper than the 64 levels CapFs walks: refused as it is made. + let r = sandbox + .run( + "import errno, os\n\ + try:\n \ + os.makedirs('/output/' + '/'.join(['d'] * 70))\n\ + except OSError as e:\n \ + print(errno.errorcode[e.errno])", + ) + .unwrap(); + assert_eq!( + (r.exit_code, r.stdout.as_str()), + (0, "EINVAL\n"), + "{}", + r.stderr + ); + std::fs::remove_dir_all(output.join("d")).unwrap(); -#[test] -fn hard_links_in_output_are_removed() { - let Some(mut sandbox) = python_sandbox(ToolRegistry::new(), None) else { - return; - }; + // More entries than the 100,000 it walks, directories counting: + // refused from the one past them. let r = sandbox - .run("import os\nopen('/output/a', 'w').write('a')\nos.link('/output/a', '/output/b')") + .run( + "import errno, os\n\ + try:\n \ + for i in range(100100):\n \ + os.mkdir(f'/output/{i}')\n\ + except OSError as e:\n \ + print(i, errno.errorcode[e.errno])", + ) .unwrap(); - assert_eq!(r.exit_code, 1, "{}", r.stderr); - assert!(r.stderr.contains("hard-linked"), "{}", r.stderr); - assert!(sandbox.get_output_files().unwrap().is_empty()); + assert_eq!( + (r.exit_code, r.stdout.as_str()), + (0, "100000 EDQUOT\n"), + "{}", + r.stderr + ); + for entry in std::fs::read_dir(&output).unwrap() { + std::fs::remove_dir(entry.unwrap().path()).unwrap(); + } + // The next run is unaffected. + let r = sandbox.run("open('/output/ok', 'w').write('ok')").unwrap(); + assert_eq!(r.exit_code, 0, "{}", r.stderr); + assert_eq!(sandbox.get_output_files().unwrap(), vec!["ok".to_string()]); } #[test] @@ -545,8 +757,17 @@ fn host_directories_in_output_are_kept() { }; let output = tempfile::tempdir().unwrap(); std::fs::create_dir_all(output.path().join("plots/png")).unwrap(); + let host_side = output.path().to_owned(); + let mut tools = ToolRegistry::new(); + tools.register_unvalidated("spoil", move |_| { + std::fs::write(host_side.join("big.bin"), vec![0u8; 2048]).unwrap(); + Ok(serde_json::Value::Null) + }); + let limits = hyperlight_sandbox::FilesystemLimits::new(1024, 8192, 8).unwrap(); let mut sandbox = SandboxBuilder::new() .module_path(rootfs) + .with_tools(tools) + .filesystem_limits(limits) .output_dir( output.path(), hyperlight_sandbox::DirPerms::READ | hyperlight_sandbox::DirPerms::MUTATE, @@ -573,7 +794,7 @@ fn host_directories_in_output_are_kept() { // A run that fails the check clears it: the directories it started // with stay, emptied; the ones it made go. let r = sandbox - .run("import os\nos.makedirs('/output/during/run')\nos.symlink('x', '/output/link')") + .run("import os\nos.makedirs('/output/during/run')\ncall_tool('spoil')") .unwrap(); assert_eq!(r.exit_code, 1, "{}", r.stderr); assert!(output.path().join("plots/png").is_dir()); @@ -582,7 +803,7 @@ fn host_directories_in_output_are_kept() { assert!(!output.path().join("guest/deep/f").exists()); assert!(output.path().join("staged").is_dir()); assert!(!output.path().join("during").exists()); - assert!(!output.path().join("link").exists()); + assert!(!output.path().join("big.bin").exists()); } #[test] @@ -708,9 +929,8 @@ fn native_network_reaches_allowed_names_by_name() { ); } -/// The guest keeps the `/output` it booted onto, which is the directory -/// `CapFs` checks: replacing the path on the host afterwards sends neither -/// side to the new one. +/// The guest's `/output` is the directory `CapFs` opened, which serves it: +/// replacing the path on the host afterwards sends neither to the new one. #[test] fn a_replaced_output_directory_changes_nothing() { let Some(rootfs) = rootfs("python") else { @@ -741,11 +961,8 @@ fn a_replaced_output_directory_changes_nothing() { assert_eq!(r.exit_code, 0, "{}", r.stderr); assert!(kept.join("a.txt").is_file()); assert_eq!(std::fs::read_dir(&output).unwrap().count(), 0); - // And what breaks the rules there is caught and cleared there. - let r = sandbox - .run("import os\nos.symlink('x', '/output/link')") - .unwrap(); - assert_eq!(r.exit_code, 1, "{}", r.stderr); - assert!(r.stderr.contains("symbolic links"), "{}", r.stderr); + // And it is the one cleared. + let snapshot = sandbox.snapshot().unwrap(); + sandbox.restore(&snapshot).unwrap(); assert_eq!(std::fs::read_dir(&kept).unwrap().count(), 0); } diff --git a/src/unikraft_sandbox/tests/relative_output.rs b/src/unikraft_sandbox/tests/relative_output.rs index e919ca5..291cf1c 100644 --- a/src/unikraft_sandbox/tests/relative_output.rs +++ b/src/unikraft_sandbox/tests/relative_output.rs @@ -46,12 +46,9 @@ fn relative_output_dir_ignores_later_chdir() { .unwrap(); assert_eq!(r.exit_code, 0, "{}", r.stderr); assert!(a.join("out/x.txt").is_file()); - // A symlink the guest leaves fails the run, and the clear that - // follows acts on a/out alone. - let r = sandbox - .run("import os\nos.symlink('x', '/output/link')") - .unwrap(); - assert_eq!(r.exit_code, 1, "{}", r.stderr); + // A restore clears the output, a/out alone. + let snapshot = sandbox.snapshot().unwrap(); + sandbox.restore(&snapshot).unwrap(); assert!(!a.join("out/x.txt").exists()); assert_eq!(std::fs::read_dir(a.join("out")).unwrap().count(), 0); assert_eq!(