From dd4a38a6107550860089e565a35d53915e9f2415 Mon Sep 17 00:00:00 2001 From: Fantix King Date: Sun, 8 May 2022 09:55:41 -0400 Subject: [PATCH] WIP resolver --- MANIFEST.in | 3 + resolver/Cargo.lock | 701 +++++++++++++++++++++++++++++++++++ resolver/Cargo.toml | 19 + resolver/src/kloop.rs | 53 +++ resolver/src/lib.rs | 18 + resolver/src/poller.rs | 57 +++ resolver/src/resolve.rs | 185 +++++++++ resolver/src/runtime.rs | 209 +++++++++++ setup.py | 29 +- src/kloop/fileio.pxd | 19 + src/kloop/fileio.pyx | 116 ++++++ src/kloop/includes/libc.pxd | 24 +- src/kloop/includes/linux.pxd | 4 + src/kloop/loop.pxd | 5 + src/kloop/loop.pyx | 41 +- src/kloop/resolver.pxd | 75 ++++ src/kloop/resolver.pyx | 175 +++++++++ src/kloop/tcp.pxd | 5 +- src/kloop/tcp.pyx | 26 +- src/kloop/udp.pxd | 18 + src/kloop/udp.pyx | 69 ++++ src/kloop/uring.pyx | 94 ++++- 22 files changed, 1900 insertions(+), 45 deletions(-) create mode 100644 resolver/Cargo.lock create mode 100644 resolver/Cargo.toml create mode 100644 resolver/src/kloop.rs create mode 100644 resolver/src/lib.rs create mode 100644 resolver/src/poller.rs create mode 100644 resolver/src/resolve.rs create mode 100644 resolver/src/runtime.rs create mode 100644 src/kloop/fileio.pxd create mode 100644 src/kloop/fileio.pyx create mode 100644 src/kloop/resolver.pxd create mode 100644 src/kloop/resolver.pyx create mode 100644 src/kloop/udp.pxd create mode 100644 src/kloop/udp.pyx diff --git a/MANIFEST.in b/MANIFEST.in index 834c0fc..1142877 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,8 @@ include Makefile include architecture.png recursive-include src *.pyx *.pxd *.h +include resolver/Cargo.toml +include resolver/Cargo.lock +recursive-include resolver *.rs graft tests global-exclude *.py[cod] *.c diff --git a/resolver/Cargo.lock b/resolver/Cargo.lock new file mode 100644 index 0000000..0b29308 --- /dev/null +++ b/resolver/Cargo.lock @@ -0,0 +1,701 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + +[[package]] +name = "enum-as-inner" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "ipconfig" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" +dependencies = [ + "socket2", + "widestring", + "winapi", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.124" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mio" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "ntapi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f" +dependencies = [ + "winapi", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "proc-macro2" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "resolver" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures-executor", + "futures-io", + "futures-util", + "libc", + "log", + "resolv-conf", + "trust-dns-proto", + "trust-dns-resolver", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f48b6d60512a392e34dbf7fd456249fd2de3c83669ab642e021903f4015185b" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "socket2", + "winapi", +] + +[[package]] +name = "trust-dns-proto" +version = "0.21.2" +source = "git+https://github.com/fantix/trust-dns?branch=0_21_2_patch#843c22f1d267660d816480d7168c17ee5467cda8" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.21.2" +source = "git+https://github.com/fantix/trust-dns?branch=0_21_2_patch#843c22f1d267660d816480d7168c17ee5467cda8" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "parking_lot", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "widestring" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" + +[[package]] +name = "windows_i686_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" + +[[package]] +name = "windows_i686_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] diff --git a/resolver/Cargo.toml b/resolver/Cargo.toml new file mode 100644 index 0000000..15ea409 --- /dev/null +++ b/resolver/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "resolver" +version = "0.1.0" +edition = "2021" + +[lib] +name = "kloop_resolver" +crate-type = ["staticlib"] + +[dependencies] +libc = "0.2.124" +log = "0.4.16" +resolv-conf = { version = "0.7.0", features = ["system"] } +trust-dns-proto = { git = "https://github.com/fantix/trust-dns", branch = "0_21_2_patch", default-features = true } +trust-dns-resolver = { git = "https://github.com/fantix/trust-dns", branch = "0_21_2_patch", default-features = true } +futures-util = "0.3.21" +futures-io = "0.3.5" +futures-executor = "0.3.5" +async-trait = "0.1.43" diff --git a/resolver/src/kloop.rs b/resolver/src/kloop.rs new file mode 100644 index 0000000..a9c91db --- /dev/null +++ b/resolver/src/kloop.rs @@ -0,0 +1,53 @@ +/* +Copyright (c) 2022 Fantix King http://fantix.pro +kLoop is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + + +use core::marker; +use std::task::Waker; +use libc; + +use crate::resolve::KLoopResolver; + +#[repr(C)] +pub struct CResolver { + _data: [u8; 0], + _marker: marker::PhantomData<(*mut u8, marker::PhantomPinned)>, +} + +#[repr(C)] +pub struct CResolve { + _data: [u8; 0], + _marker: marker::PhantomData<(*mut u8, marker::PhantomPinned)>, +} + +#[repr(C)] +pub struct UDPSend { + _data: [u8; 0], + _marker: marker::PhantomData<(*mut u8, marker::PhantomPinned)>, +} + +extern "C" { + pub fn resolver_set(c_resolver: *const CResolver, resolver: *mut KLoopResolver); + // pub fn resolve_set_poller(resolve: *const CResolve, poller: *mut Poller); + pub fn resolve_prep_addr(resolve: *const CResolve) -> *mut libc::sockaddr; + pub fn resolve_done_cb(resolve: *const CResolve) -> libc::c_int; + pub fn udp_bind(addr: *const libc::sockaddr, addrlen: libc::socklen_t) -> libc::c_int; + pub fn udp_send_init(fd: libc::c_int, resolver: *const CResolver) -> libc::c_ulong; + pub fn udp_send_poll( + send: libc::c_ulong, + data: *const u8, + datalen: libc::size_t, + addr: *const libc::sockaddr, + addrlen: libc::socklen_t, + waker: *mut Waker, + ) -> libc::c_int; +} diff --git a/resolver/src/lib.rs b/resolver/src/lib.rs new file mode 100644 index 0000000..e0ef8ad --- /dev/null +++ b/resolver/src/lib.rs @@ -0,0 +1,18 @@ +/* +Copyright (c) 2022 Fantix King http://fantix.pro +kLoop is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + + +mod kloop; +mod resolve; +mod runtime; + +pub use resolve::{resolver_init, resolver_lookup_ip}; diff --git a/resolver/src/poller.rs b/resolver/src/poller.rs new file mode 100644 index 0000000..f930def --- /dev/null +++ b/resolver/src/poller.rs @@ -0,0 +1,57 @@ +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use futures_util::task::noop_waker_ref; + +use libc; + +pub type BoxFuture<'a, T> = Box + 'a>; + +pub struct Poller<'a> { + ctx: Context<'a>, + fut: BoxFuture<'a, libc::c_int>, +} + +impl<'a> Poller<'a> { + pub fn new(fut: impl Future + 'a) -> Self { + let waker = noop_waker_ref(); + Poller { + ctx: Context::from_waker(waker), + fut: Box::new(fut), + } + } + + pub fn step(&mut self) -> Option { + let fut = unsafe { Pin::new_unchecked(self.fut.as_mut()) }; + match fut.poll(&mut self.ctx) { + Poll::Ready(rv) => Some(rv), + Poll::Pending => None, + } + } +} + +pub struct OnceFuture { + seen: bool, +} + +impl OnceFuture { + pub fn new() -> Self { + Self { seen: false } + } + +} + +impl Future for OnceFuture { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.seen { + println!("poll ready"); + Poll::Ready(()) + } else { + println!("poll pending"); + self.get_mut().seen = true; + Poll::Pending + } + } +} diff --git a/resolver/src/resolve.rs b/resolver/src/resolve.rs new file mode 100644 index 0000000..99d2a32 --- /dev/null +++ b/resolver/src/resolve.rs @@ -0,0 +1,185 @@ +/* +Copyright (c) 2022 Fantix King http://fantix.pro +kLoop is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + + +use futures_executor::{LocalPool, LocalSpawner}; +use futures_util::task::{LocalSpawnExt, SpawnError}; +use std; +use std::cell::RefCell; +use std::future::Future; +use std::io; +use std::net::IpAddr; +use std::rc::Rc; + +use libc; +use trust_dns_resolver::name_server::{GenericConnection, GenericConnectionProvider}; +use trust_dns_resolver::system_conf::parse_resolv_conf; +use trust_dns_resolver::{AsyncResolver, Hosts}; + +use crate::kloop::{resolve_done_cb, resolve_prep_addr, resolver_set, CResolve, CResolver}; +use crate::runtime::{KLoopHandle, KLoopRuntime}; + +const DEFAULT_PORT: u16 = 53; + +type KLoopConnection = GenericConnection; + +type KLoopConnectionProvider = GenericConnectionProvider; + +thread_local! { + pub static CURRENT_RESOLVER: Rc>> = Rc::new(RefCell::new(None)); +} + +#[derive(Debug)] +pub struct KLoopResolver { + resolver: AsyncResolver, + pub c_resolver: *const CResolver, + pool: LocalPool, + spawner: LocalSpawner, +} + +impl KLoopResolver { + fn new( + resolv_conf: &[u8], + hosts_conf: &[u8], + c_resolver: *const CResolver, + ) -> io::Result { + let (config, mut options) = parse_resolv_conf(resolv_conf)?; + options.use_hosts_file = false; + let conn_provider = GenericConnectionProvider::new(KLoopHandle); + let mut resolver = AsyncResolver::new_with_conn(config, options, conn_provider).unwrap(); + resolver.set_hosts(Some(Hosts::default().read_hosts_conf(hosts_conf))); + let pool = LocalPool::new(); + let spawner = pool.spawner(); + Ok(Self { + resolver, + c_resolver, + pool, + spawner, + }) + } + + pub fn spawn(&self, future: Fut) -> Result<(), SpawnError> + where + Fut: Future + 'static, + { + self.spawner.spawn_local(future) + } + + fn run_until_stalled(&mut self) { + loop { + self.pool.run_until_stalled(); + if !self.pool.try_run_one() { + break; + } + } + } + + async fn lookup_ip(&self, resolve: *mut CResolve, host: &str, port: libc::in_port_t) -> () { + match self.resolver.lookup_ip(host).await { + Ok(result) => { + for ip in result.into_iter() { + match ip { + IpAddr::V4(ip) => unsafe { + let out = resolve_prep_addr(resolve) as *mut libc::sockaddr_in; + if out.is_null() { + println!("resolve_prep_addr returned NULL"); + break; + } + (*out).sin_family = libc::AF_INET as libc::sa_family_t; + (*out).sin_addr = libc::in_addr { + s_addr: u32::from_ne_bytes(ip.octets()), + }; + (*out).sin_port = port.to_be(); + (*out).sin_zero = [0; 8]; + }, + IpAddr::V6(ip) => unsafe { + let out = resolve_prep_addr(resolve) as *mut libc::sockaddr_in6; + if out.is_null() { + println!("resolve_prep_addr returned NULL"); + break; + } + (*out).sin6_family = libc::AF_INET6 as libc::sa_family_t; + (*out).sin6_addr = libc::in6_addr { + s6_addr: ip.octets(), + }; + (*out).sin6_port = port.to_be(); + }, + } + } + } + Err(e) => { + println!("lookup_ip error: {:?}", e); + } + } + unsafe { + resolve_done_cb(resolve); + } + } +} + +#[no_mangle] +pub extern "C" fn resolver_init( + c_resolver: *const CResolver, + resolv_conf_data: *const u8, + resolv_conf_data_size: libc::size_t, + hosts_conf_data: *const u8, + hosts_conf_data_size: libc::size_t, +) -> libc::c_int { + let resolv_conf = + unsafe { std::slice::from_raw_parts(resolv_conf_data, resolv_conf_data_size) }; + let hosts_conf = unsafe { std::slice::from_raw_parts(hosts_conf_data, hosts_conf_data_size) }; + let mut resolver = match KLoopResolver::new(resolv_conf, hosts_conf, c_resolver) { + Ok(resolver) => resolver, + Err(e) => return 0, + }; + let rv = Box::into_raw(Box::new(resolver)); + unsafe { + resolver_set(c_resolver, rv); + } + 1 +} + +#[no_mangle] +pub extern "C" fn resolver_lookup_ip( + resolver: *mut KLoopResolver, + resolve: *mut CResolve, + host_raw: *const u8, + length: libc::size_t, + port: libc::in_port_t, +) -> libc::c_int { + let host = match std::str::from_utf8(unsafe { std::slice::from_raw_parts(host_raw, length) }) { + Ok(host) => host, + _ => return 0, + }; + let r = || unsafe { resolver.as_mut() }.unwrap(); + + let fut = r().lookup_ip(resolve, host, port); + if let Err(e) = r().spawn(fut) { + println!("spawn error: {:?}", e); + return 0; + } + CURRENT_RESOLVER.with(|current| { + *current.borrow_mut() = Some(resolver); + r().run_until_stalled(); + *current.borrow_mut() = None; + }); + 1 +} + +#[no_mangle] +pub extern "C" fn resolver_run_until_stalled(resolver: *mut KLoopResolver) { + CURRENT_RESOLVER.with(|current| { + *current.borrow_mut() = Some(resolver); + unsafe { resolver.as_mut() }.unwrap().run_until_stalled(); + *current.borrow_mut() = None; + }); +} diff --git a/resolver/src/runtime.rs b/resolver/src/runtime.rs new file mode 100644 index 0000000..1bad4a7 --- /dev/null +++ b/resolver/src/runtime.rs @@ -0,0 +1,209 @@ +/* +Copyright (c) 2022 Fantix King http://fantix.pro +kLoop is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + http://license.coscl.org.cn/MulanPSL2 +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +*/ + + +use crate::kloop; +use crate::resolve::CURRENT_RESOLVER; +use async_trait::async_trait; +use futures_io::{AsyncRead, AsyncWrite}; +use libc::{sockaddr, socklen_t}; +use std::fmt::Debug; +use std::future::Future; +use std::net::SocketAddr; +use std::pin::Pin; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; +use std::{io, mem}; +use trust_dns_proto::error::ProtoError; +use trust_dns_proto::tcp::{Connect, DnsTcpStream}; +use trust_dns_proto::udp::UdpSocket; +use trust_dns_proto::Time; +use trust_dns_resolver::name_server::{RuntimeProvider, Spawn}; + +#[derive(Clone, Copy, Debug)] +pub struct KLoopTimer {} + +#[async_trait] +impl Time for KLoopTimer { + async fn delay_for(duration: Duration) { + println!("TODO: delay_for: {:?}", duration); + } + + async fn timeout( + duration: Duration, + future: F, + ) -> io::Result { + println!("TODO: timeout: {:?}", duration); + Ok(future.await) + } +} + +pub struct KLoopTcp {} + +#[async_trait] +impl Connect for KLoopTcp { + async fn connect_with_bind( + addr: SocketAddr, + bind_addr: Option, + ) -> io::Result { + println!("TODO: connect_with_bind: {:?} {:?}", addr, bind_addr); + todo!() + } +} + +impl AsyncRead for KLoopTcp { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + println!("TODO: poll_read"); + todo!() + } +} + +impl AsyncWrite for KLoopTcp { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + println!("TODO: poll_write"); + todo!() + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + println!("TODO: poll_flush"); + todo!() + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + println!("TODO: poll_close"); + todo!() + } +} + +impl DnsTcpStream for KLoopTcp { + type Time = KLoopTimer; +} + +pub struct KLoopUdp { + fd: libc::c_int, + send: libc::c_ulong, +} + +#[async_trait] +impl UdpSocket for KLoopUdp { + type Time = KLoopTimer; + + async fn bind(addr: SocketAddr) -> io::Result { + let (addr_ptr, addr_len) = socket_addr_as_ptr(addr); + let fd = unsafe { kloop::udp_bind(addr_ptr, addr_len) }; + CURRENT_RESOLVER.with(|resolver| { + let resolver = resolver.borrow().unwrap(); + let resolver = unsafe { resolver.as_ref() }.unwrap(); + let resolver = resolver.c_resolver; + let send = unsafe { kloop::udp_send_init(fd, resolver) }; + Ok(KLoopUdp { fd, send }) + }) + } + + fn poll_recv_from( + &self, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + println!("TODO: poll_recv_from"); + todo!() + } + + fn poll_send_to( + &self, + cx: &mut Context<'_>, + buf: &[u8], + target: SocketAddr, + ) -> Poll> { + let waker = Box::new(cx.waker().clone()); + let (addr, addrlen) = socket_addr_as_ptr(target); + match unsafe { + kloop::udp_send_poll( + self.send, + buf.as_ptr(), + buf.len(), + addr, + addrlen, + Box::into_raw(waker), + ) + } { + res if res > 0 => { + Poll::Ready(Ok(res as usize)) + } + res => { + Poll::Pending + } + } + } +} + +fn socket_addr_as_ptr(addr: SocketAddr) -> (*const sockaddr, socklen_t) { + match addr { + SocketAddr::V4(ref a) => ( + a as *const _ as *const _, + mem::size_of_val(a) as libc::socklen_t, + ), + SocketAddr::V6(ref a) => ( + a as *const _ as *const _, + mem::size_of_val(a) as libc::socklen_t, + ), + } +} + +#[derive(Clone, Copy)] +pub struct KLoopHandle; + +impl Spawn for KLoopHandle { + fn spawn_bg(&mut self, future: F) + where + F: Future> + Send + 'static, + { + CURRENT_RESOLVER.with(|resolver| { + let r = resolver.borrow().unwrap(); + let r = unsafe { r.as_mut() }.unwrap(); + r.spawn(async { + future.await.unwrap_or_else(|e| { + println!("spawn_bg error: {:?}", e); + }); + }); + }); + } +} + +#[derive(Clone, Copy)] +pub struct KLoopRuntime; + +impl RuntimeProvider for KLoopRuntime { + type Handle = KLoopHandle; + type Timer = KLoopTimer; + type Udp = KLoopUdp; + type Tcp = KLoopTcp; +} + +#[no_mangle] +pub extern "C" fn waker_wake(waker: *mut Waker) { + let waker = unsafe { Box::from_raw(waker) }; + waker.wake(); +} + +#[no_mangle] +pub extern "C" fn waker_forget(waker: *mut Waker) { + unsafe { Box::from_raw(waker) }; +} diff --git a/setup.py b/setup.py index 2c76f65..26070cc 100644 --- a/setup.py +++ b/setup.py @@ -9,17 +9,42 @@ # See the Mulan PSL v2 for more details. +import subprocess import sysconfig from setuptools import setup from Cython.Build import cythonize from Cython.Distutils import Extension +from Cython.Distutils import build_ext + + +DEBUG = True +RESOLVER_LIB = ( + f"resolver/target/{'debug' if DEBUG else 'release'}/libkloop_resolver.a" +) + + +class build_ext_with_resolver(build_ext): + def run(self): + subprocess.check_call( + ["cargo", "build"] + [] if DEBUG else ["-r"], + cwd="resolver", + ) + super().run() setup( + cmdclass={ + "build_ext": build_ext_with_resolver, + }, ext_modules=cythonize( [ - Extension("kloop.loop", ["src/kloop/loop.pyx"]), + Extension( + "kloop.loop", + ["src/kloop/loop.pyx"], + extra_link_args=[RESOLVER_LIB], + depends=[RESOLVER_LIB], + ), Extension( "kloop.ktls", ["src/kloop/ktls.pyx"], @@ -53,5 +78,5 @@ setup( ), ], language_level="3", - ) + ), ) diff --git a/src/kloop/fileio.pxd b/src/kloop/fileio.pxd new file mode 100644 index 0000000..b6d2a7d --- /dev/null +++ b/src/kloop/fileio.pxd @@ -0,0 +1,19 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef struct FileReader: + Loop* loop + const char* path + int fd, cancelled + char* data + size_t size, offset + RingCallback ring_cb + RingCallback done_cb diff --git a/src/kloop/fileio.pyx b/src/kloop/fileio.pyx new file mode 100644 index 0000000..9be86b9 --- /dev/null +++ b/src/kloop/fileio.pyx @@ -0,0 +1,116 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef size_t FILE_READ_BUF_SIZE = 4096 + + +cdef int file_reader_openat_cb(RingCallback* cb) nogil except 0: + cdef: + int fd = cb.res + FileReader* fr = cb.data + + if fd < 0: + fr.done_cb.res = fd + return fr.done_cb.callback(&fr.done_cb) + if fr.cancelled: + fr.done_cb.res = -libc.ECANCELED + return fr.done_cb.callback(&fr.done_cb) + fr.fd = fd + fr.data = PyMem_RawMalloc(FILE_READ_BUF_SIZE) + if fr.data == NULL: + fr.done_cb.res = -errno.ENOMEM + return fr.done_cb.callback(&fr.done_cb) + fr.size = FILE_READ_BUF_SIZE + fr.offset = 0 + fr.ring_cb.callback = file_reader_read_cb + if ring_sq_submit_read( + &fr.loop.ring.sq, + fd, + fr.data, + FILE_READ_BUF_SIZE, + 0, + &fr.ring_cb, + ): + return 1 + else: + fr.done_cb.res = -errno.EAGAIN + return fr.done_cb.callback(&fr.done_cb) + + +cdef int file_reader_read_cb(RingCallback* cb) nogil except 0: + cdef: + int read = cb.res + FileReader* fr = cb.data + size_t size = fr.size + size_t offset = fr.offset + + if read < 0: + fr.done_cb.res = read + return fr.done_cb.callback(&fr.done_cb) + if fr.cancelled: + fr.done_cb.res = -libc.ECANCELED + return fr.done_cb.callback(&fr.done_cb) + offset += read + if read > 0 and offset == size: + size += FILE_READ_BUF_SIZE + if PyMem_RawRealloc(fr.data, size) == NULL: + fr.done_cb.res = -errno.ENOMEM + return fr.done_cb.callback(&fr.done_cb) + fr.size = size + fr.offset = offset + if ring_sq_submit_read( + &fr.loop.ring.sq, + fr.fd, + fr.data + offset, + FILE_READ_BUF_SIZE, + offset, + &fr.ring_cb, + ): + return 1 + else: + fr.done_cb.res = -errno.EAGAIN + return fr.done_cb.callback(&fr.done_cb) + else: + fr.done_cb.res = 1 + fr.offset = offset + return fr.done_cb.callback(&fr.done_cb) + + +cdef int file_reader_start( + FileReader* fr, Loop* loop, const char* path +) nogil: + fr.loop = loop + fr.done_cb.res = 0 + fr.fd = 0 + fr.data = NULL + fr.cancelled = 0 + fr.ring_cb.callback = file_reader_openat_cb + fr.ring_cb.data = fr + return ring_sq_submit_openat( + &loop.ring.sq, + 0, # dfd + path, # path + 0, # flags + 0, # mode + &fr.ring_cb + ) + + +cdef int file_reader_done(FileReader* fr) nogil: + cdef int fd = fr.fd + fr.done_cb.res = 0 + if fr.data != NULL: + PyMem_RawFree(fr.data) + fr.data = NULL + if fd != 0: + fr.fd = 0 + return ring_sq_submit_close(&fr.loop.ring.sq, fd, NULL) + return 1 \ No newline at end of file diff --git a/src/kloop/includes/libc.pxd b/src/kloop/includes/libc.pxd index 7a50081..6ba1fe6 100644 --- a/src/kloop/includes/libc.pxd +++ b/src/kloop/includes/libc.pxd @@ -21,18 +21,22 @@ cdef extern from "signal.h" nogil: int _NSIG cdef extern from "sys/socket.h" nogil: + enum: + SOCK_DGRAM + SOCK_STREAM + ctypedef int socklen_t + ctypedef int sa_family_t + ctypedef int in_port_t + int SOL_TLS - struct in_addr: - pass - - struct sockaddr_in: - int sin_family - int sin_port - in_addr sin_addr + struct sockaddr: + sa_family_t sa_family struct msghdr: + void* msg_name # Optional address + socklen_t msg_namelen # Size of address iovec* msg_iov # Scatter/gather array size_t msg_iovlen # Number of elements in msg_iov void* msg_control # ancillary data, see below @@ -57,6 +61,7 @@ cdef extern from "sys/socket.h" nogil: const void* option_value, socklen_t option_len, ) + int bind(int sockfd, const sockaddr* addr, socklen_t addrlen) cdef extern from "arpa/inet.h" nogil: @@ -67,3 +72,8 @@ cdef extern from "sys/uio.h" nogil: struct iovec: void* iov_base size_t iov_len + + +cdef extern from "" nogil: + enum: + ECANCELED diff --git a/src/kloop/includes/linux.pxd b/src/kloop/includes/linux.pxd index c0b564b..f4498bd 100644 --- a/src/kloop/includes/linux.pxd +++ b/src/kloop/includes/linux.pxd @@ -80,6 +80,9 @@ cdef extern from "linux/io_uring.h" nogil: IORING_OP_SENDMSG IORING_OP_RECV IORING_OP_RECVMSG + IORING_OP_OPENAT + IORING_OP_READ + IORING_OP_CLOSE struct io_sqring_offsets: __u32 head @@ -120,6 +123,7 @@ cdef extern from "linux/io_uring.h" nogil: __u32 len # buffer size or number of iovecs __u64 user_data # data to be passed back at completion time __u8 flags # IOSQE_ flags + __u32 open_flags struct io_uring_cqe: __u64 user_data # data to be passed back at completion time diff --git a/src/kloop/loop.pxd b/src/kloop/loop.pxd index 86bc8f2..b5c5192 100644 --- a/src/kloop/loop.pxd +++ b/src/kloop/loop.pxd @@ -14,6 +14,7 @@ from cpython cimport PyMem_RawMalloc, PyMem_RawFree, PyMem_RawRealloc from cpython cimport PyObject, Py_INCREF, Py_DECREF from libc cimport errno, string from posix cimport mman, unistd, time +from posix.types cimport mode_t from .includes cimport libc, linux, barrier @@ -22,6 +23,9 @@ include "./queue.pxd" include "./heapq.pxd" include "./uring.pxd" include "./tcp.pxd" +include "./udp.pxd" +include "./fileio.pxd" +include "./resolver.pxd" cdef struct Loop: @@ -31,6 +35,7 @@ cdef struct Loop: Queue ready int timer_cancelled_count PyObject* loop + CResolver resolver cdef class KLoopImpl: diff --git a/src/kloop/loop.pyx b/src/kloop/loop.pyx index 613fc1b..92c0dcf 100644 --- a/src/kloop/loop.pyx +++ b/src/kloop/loop.pyx @@ -16,7 +16,6 @@ import functools import inspect import os import reprlib -import socket import threading import traceback @@ -39,11 +38,14 @@ cdef int MIN_SCHEDULED_TIMER_HANDLES = 100 # that are cancelled before cleanup cdef int MAX_CANCELLED_TIMER_HANDLES_RATIO = 2 -include "./handle.pyx" -include "./queue.pyx" -include "./heapq.pyx" -include "./uring.pyx" -include "./tcp.pyx" +include "handle.pyx" +include "queue.pyx" +include "heapq.pyx" +include "uring.pyx" +include "tcp.pyx" +include "udp.pyx" +include "fileio.pyx" +include "resolver.pyx" cdef long long monotonic_ns() nogil except -1: @@ -172,7 +174,7 @@ cdef inline int loop_run_once( Callback* callback long long timeout = -1, now int nready - RingCallback* cb + RingCallback* cb = NULL if scheduled.tail: if not filter_cancelled_calls(loop): @@ -248,6 +250,7 @@ cdef class KLoopImpl: with nogil: loop_init(&self.loop, depth, ¶ms) + self.resolver = Resolver.new(self) self.closed = False self.thread_id = None @@ -510,11 +513,25 @@ cdef class KLoopImpl: happy_eyeballs_delay=None, interleave=None, ): - cdef TCPTransport transport = TCPTransport.new(protocol_factory, self) - r = socket.getaddrinfo(host, port)[0] - host, port = r[-1] - waiter = transport.connect(host, port) - return transport, await waiter + cdef: + TCPTransport transport + Resolve resolve + object waiter + size_t i + + resolve = await self.resolver.lookup_ip(host, port) + if not resolve.r.result_len: + raise RuntimeError(f"Cannot resolve host: {host!r}") + + transport = TCPTransport.new(protocol_factory, self) + exceptions = [] + for i in range(resolve.r.result_len): + try: + waiter = transport.connect(resolve.r.result + i) + return transport, await waiter + except OSError as e: + exceptions.append(e) + raise exceptions[0] class KLoop(KLoopImpl, asyncio.AbstractEventLoop): diff --git a/src/kloop/resolver.pxd b/src/kloop/resolver.pxd new file mode 100644 index 0000000..eb6c145 --- /dev/null +++ b/src/kloop/resolver.pxd @@ -0,0 +1,75 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef extern from * nogil: + int resolver_init( + CResolver* resolver, + char* resolv_conf_data, + size_t resolv_conf_data_size, + char* hosts_conf_data, + size_t hosts_conf_data_size, + ) + int resolver_lookup_ip( + void* resolver, + void* resolve, + char* host, + size_t length, + libc.in_port_t port, + ) + void resolver_run_until_stalled(void* rust_resolver) + void waker_wake(void* waker) + void waker_forget(void* waker) + + +cdef struct CResolver: + Loop* loop + Callback* cb + FileReader resolv_conf + FileReader hosts_conf + int res + void* rust_resolver + + +cdef class Resolver: + cdef: + CResolver resolver + KLoopImpl loop + Handle handle + object waiter + bint initialized + + @staticmethod + cdef Resolver new(KLoopImpl loop) + cdef init_cb(self) + cdef err_cb(self, exc) + + +cdef struct CResolve: + CResolver* resolver + libc.sockaddr* result + size_t result_len, result_size + Callback* cb + int res + char* host + size_t host_len + libc.in_port_t port + + +cdef class Resolve: + cdef: + CResolve r + Handle handle + object waiter + object host + + @staticmethod + cdef new(Resolver resolver, host, port) + cdef resolve_cb(self) diff --git a/src/kloop/resolver.pyx b/src/kloop/resolver.pyx new file mode 100644 index 0000000..dd3b96b --- /dev/null +++ b/src/kloop/resolver.pyx @@ -0,0 +1,175 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef const char* RESOLV_CONF = "/etc/resolv.conf" +cdef const char* HOSTS_CONF = "/etc/hosts" +cdef size_t SOCKADDR_CHUNK_SIZE = 4 + + +cdef int resolve_cb(RingCallback* cb) nogil except 0: + cdef: + CResolver* r = cb.data + int rv = 1 + + return rv + + +cdef int resolver_read_file_cb(RingCallback* cb) nogil except 0: + cdef: + CResolver* r = cb.data + int rv = 1 + void* ptr + + if r.hosts_conf.done_cb.res == 0 or r.resolv_conf.done_cb.res == 0: + if cb.res < 0: + r.resolv_conf.cancelled = r.hosts_conf.cancelled = 1 + else: + r.res = min(r.hosts_conf.done_cb.res, r.resolv_conf.done_cb.res) + if r.res > 0: + r.res = resolver_init( + r, + r.resolv_conf.data, + r.resolv_conf.offset, + r.hosts_conf.data, + r.hosts_conf.offset, + ) + rv = queue_push(&r.loop.ready, r.cb) + if not file_reader_done(&r.resolv_conf): + # TODO: fd not closed? + pass + if not file_reader_done(&r.hosts_conf): + # TODO: fd not closed? + pass + return rv + + +cdef extern libc.sockaddr* resolve_prep_addr(CResolve* r) nogil: + cdef size_t l = r.result_len, size = r.result_size + if l == size: + size += SOCKADDR_CHUNK_SIZE + if PyMem_RawRealloc(r.result, sizeof(libc.sockaddr) * size) == NULL: + return NULL + r.result_size = size + r.result_len = l + 1 + return r.result + l + + +cdef extern int resolve_done_cb(CResolve* r) nogil: + return queue_push(&r.resolver.loop.ready, r.cb) + + +cdef extern void resolver_set(CResolver* resolver, void* rust_resolver) nogil: + resolver.rust_resolver = rust_resolver + + +cdef class Resolver: + @staticmethod + cdef Resolver new(KLoopImpl loop): + cdef: + Resolver rv = Resolver.__new__(Resolver) + CResolver* r = &rv.resolver + rv.loop = loop + r.loop = &loop.loop + r.resolv_conf.done_cb.callback = resolver_read_file_cb + r.resolv_conf.done_cb.data = r + r.hosts_conf.done_cb.callback = resolver_read_file_cb + r.hosts_conf.done_cb.data = r + return rv + + async def ensure_initialized(self): + cdef CResolver* r + + if self.initialized: + return + waiter = self.waiter + if waiter is None: + r = &self.resolver + waiter = self.waiter = self.loop.create_future() + if not file_reader_start(&r.resolv_conf, r.loop, RESOLV_CONF): + self.err_cb(ValueError("Submission queue is full!")) + elif not file_reader_start(&r.hosts_conf, r.loop, HOSTS_CONF): + r.resolv_conf.cancelled = 1 + self.err_cb(ValueError("Submission queue is full!")) + else: + self.handle = Handle(self.init_cb, (self,), self.loop, None) + r.cb = &self.handle.cb + await waiter + + async def reload_config(self, *, force=False): + if self.initialized: + waiter = self.waiter + if waiter is None: + waiter = self.waiter = self.loop.create_future() + self.err_cb(NotImplementedError()) + await waiter + else: + await self.ensure_initialized() + + cdef init_cb(self): + cdef int res = self.resolver.res + + if res < 0: + try: + errno.errno = -res + PyErr_SetFromErrno(IOError) + except IOError as e: + self.waiter.set_exception(e) + else: + self.waiter.set_result(None) + + cdef err_cb(self, exc): + waiter, self.waiter = self.waiter, None + if waiter is not None: + waiter.set_exception(exc) + + async def lookup_ip(self, host, port): + await self.ensure_initialized() + return await Resolve.new(self, host, port) + + +cdef class Resolve: + @staticmethod + cdef new(Resolver resolver, host, port): + cdef: + Resolve rv = Resolve.__new__(Resolve) + CResolve* r = &rv.r + rv.host = host.encode("utf-8") + rv.waiter = resolver.loop.create_future() + r.resolver = &resolver.resolver + r.host = rv.host + r.host_len = len(rv.host) + r.port = port + r.result = PyMem_RawMalloc( + sizeof(libc.sockaddr) * SOCKADDR_CHUNK_SIZE + ) + if r.result == NULL: + raise MemoryError + r.result_size = SOCKADDR_CHUNK_SIZE + rv.handle = Handle(rv.resolve_cb, (rv,), resolver.loop, None) + r.cb = &rv.handle.cb + return rv + + def __await__(self): + cdef CResolve* r = &self.r + resolver_lookup_ip(r.resolver.rust_resolver, r, r.host, r.host_len, r.port) + return self.waiter.__await__() + + def __dealloc__(self): + cdef CResolve* r = &self.r + r.host = NULL + r.host_len = 0 + if r.result != NULL: + PyMem_RawFree(r.result) + r.result = NULL + r.result_size = 0 + + cdef resolve_cb(self): + self.waiter.set_result(self) diff --git a/src/kloop/tcp.pxd b/src/kloop/tcp.pxd index 3d3abe9..eb61958 100644 --- a/src/kloop/tcp.pxd +++ b/src/kloop/tcp.pxd @@ -11,7 +11,7 @@ cdef struct TCPConnect: int fd - libc.sockaddr_in addr + libc.sockaddr* addr RingCallback ring_cb Loop* loop Callback* cb @@ -24,11 +24,10 @@ cdef class TCPTransport: TCPConnect connector object waiter object protocol_factory - object host_bytes Handle handle @staticmethod cdef TCPTransport new(object protocol_factory, KLoopImpl loop) - cdef connect(self, host, port) + cdef connect(self, libc.sockaddr* addr) cdef connect_cb(self) diff --git a/src/kloop/tcp.pyx b/src/kloop/tcp.pyx index 50fb876..399ff50 100644 --- a/src/kloop/tcp.pyx +++ b/src/kloop/tcp.pyx @@ -13,7 +13,7 @@ cdef int tcp_connect(TCPConnect* connector) nogil: return ring_sq_submit_connect( &connector.loop.ring.sq, connector.fd, - &connector.addr, + connector.addr, &connector.ring_cb, ) @@ -29,29 +29,22 @@ cdef class TCPTransport: cdef TCPTransport rv = TCPTransport.__new__(TCPTransport) rv.protocol_factory = protocol_factory rv.loop = loop + rv.connector.loop = &loop.loop + rv.connector.ring_cb.callback = tcp_connect_cb + rv.connector.ring_cb.data = &rv.connector return rv - cdef connect(self, host, port): + cdef connect(self, libc.sockaddr* addr): cdef: int fd TCPConnect* c = &self.connector - fd = libc.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + fd = libc.socket(addr.sa_family, libc.SOCK_STREAM, 0) if fd == -1: PyErr_SetFromErrno(IOError) return - self.host_bytes = host.encode() - if not libc.inet_pton( - socket.AF_INET, self.host_bytes, &c.addr.sin_addr - ): - PyErr_SetFromErrno(IOError) - return - c.addr.sin_family = socket.AF_INET - c.addr.sin_port = libc.htons(port) + c.addr = addr c.fd = self.fd = fd - c.loop = &self.loop.loop - c.ring_cb.callback = tcp_connect_cb - c.ring_cb.data = c self.handle = Handle(self.connect_cb, (self,), self.loop, None) c.cb = &self.handle.cb if not tcp_connect(c): @@ -61,6 +54,11 @@ cdef class TCPTransport: cdef connect_cb(self): if self.connector.ring_cb.res != 0: + if not ring_sq_submit_close( + &self.loop.loop.ring.sq, self.fd, NULL + ): + # TODO: fd not closed? + pass try: errno.errno = abs(self.connector.ring_cb.res) PyErr_SetFromErrno(IOError) diff --git a/src/kloop/udp.pxd b/src/kloop/udp.pxd new file mode 100644 index 0000000..fb55ed9 --- /dev/null +++ b/src/kloop/udp.pxd @@ -0,0 +1,18 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef struct UDPSend: + int fd + libc.iovec vec + libc.msghdr msg + RingCallback callback + CResolver* resolver + void* rust_waker diff --git a/src/kloop/udp.pyx b/src/kloop/udp.pyx new file mode 100644 index 0000000..0c205b1 --- /dev/null +++ b/src/kloop/udp.pyx @@ -0,0 +1,69 @@ +# Copyright (c) 2022 Fantix King http://fantix.pro +# kLoop is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + +cdef extern int udp_bind(libc.sockaddr* addr, libc.socklen_t addrlen) nogil: + cdef int fd = libc.socket(addr.sa_family, libc.SOCK_DGRAM, 0) + if fd == -1: + return -1 + if libc.bind(fd, addr, addrlen) == -1: + # TODO: close fd + return -1 + return fd + + +cdef extern unsigned long udp_send_init(int fd, CResolver* resolver) nogil: + cdef UDPSend* rv + rv = PyMem_RawMalloc(sizeof(UDPSend)) + if rv == NULL: + return 0 + string.memset(rv, 0, sizeof(UDPSend)) + rv.fd = fd + rv.resolver = resolver + rv.msg.msg_iov = &rv.vec + rv.msg.msg_iovlen = 1 + rv.callback.data = rv + rv.callback.callback = udp_send_cb + return rv + + +cdef int udp_send_cb(RingCallback* cb) nogil except 0: + cdef UDPSend* send = cb.data + waker_wake(send.rust_waker) + resolver_run_until_stalled(send.resolver.rust_resolver) + return 1 + + +cdef extern int udp_send_poll( + unsigned long send_in, + char* data, + size_t datalen, + libc.sockaddr* addr, + libc.socklen_t addrlen, + void* waker, +) nogil: + cdef UDPSend* send = send_in + if send.vec.iov_base == NULL: + send.vec.iov_base = data + send.vec.iov_len = datalen + send.msg.msg_name = addr + send.msg.msg_namelen = addrlen + send.rust_waker = waker + return ring_sq_submit_sendmsg( + &send.resolver.loop.ring.sq, + send.fd, + &send.msg, + &send.callback, + ) - 1 + else: + waker_forget(waker) + if send.vec.iov_base != data or send.vec.iov_len != datalen: + return -1 + return send.callback.res or -1 diff --git a/src/kloop/uring.pyx b/src/kloop/uring.pyx index 91d75ca..9534387 100644 --- a/src/kloop/uring.pyx +++ b/src/kloop/uring.pyx @@ -176,12 +176,13 @@ cdef inline void ring_cq_pop(CompletionQueue* cq, RingCallback** callback) nogil head = cq.khead[0] cqe = cq.cqes + (head & cq.kring_mask[0]) ret = cqe.user_data - ret.res = cqe.res - callback[0] = ret + if ret != NULL: + ret.res = cqe.res + callback[0] = ret barrier.io_uring_smp_store_release(cq.khead, head + 1) -cdef inline int ring_sq_submit( +cdef inline linux.io_uring_sqe* ring_sq_submit( SubmissionQueue* sq, linux.__u8 op, int fd, @@ -209,15 +210,15 @@ cdef inline int ring_sq_submit( if link: sqe.flags = linux.IOSQE_IO_LINK sqe.user_data = callback - return 1 + return sqe else: - return 0 + return NULL cdef int ring_sq_submit_connect( - SubmissionQueue* sq, int fd, libc.sockaddr_in* addr, RingCallback* callback + SubmissionQueue* sq, int fd, libc.sockaddr* addr, RingCallback* callback ) nogil: - return ring_sq_submit( + return 1 if ring_sq_submit( sq, linux.IORING_OP_CONNECT, fd, @@ -226,4 +227,83 @@ cdef int ring_sq_submit_connect( sizeof(addr[0]), 0, callback, + ) else 0 + + +cdef int ring_sq_submit_openat( + SubmissionQueue* sq, + int dfd, + const char* path, + int flags, + mode_t mode, + RingCallback* callback, +) nogil: + cdef linux.io_uring_sqe* sqe = ring_sq_submit( + sq, + linux.IORING_OP_OPENAT, + dfd, + path, + mode, + 0, + 0, + callback, ) + if sqe == NULL: + return 0 + else: + sqe.open_flags = flags + return 1 + + +cdef int ring_sq_submit_read( + SubmissionQueue* sq, + int fd, + char* buf, + unsigned nbytes, + linux.__u64 offset, + RingCallback* callback, +) nogil: + return 1 if ring_sq_submit( + sq, + linux.IORING_OP_READ, + fd, + buf, + nbytes, + offset, + 0, + callback, + ) else 0 + + +cdef int ring_sq_submit_close( + SubmissionQueue* sq, + int fd, + RingCallback * callback, +) nogil: + return 1 if ring_sq_submit( + sq, + linux.IORING_OP_CLOSE, + fd, + 0, + 0, + 0, + 0, + callback, + ) else 0 + +cdef int ring_sq_submit_sendmsg( + SubmissionQueue* sq, + int fd, + const libc.msghdr *msg, + RingCallback* callback, +) nogil: + return 1 if ring_sq_submit( + sq, + linux.IORING_OP_SENDMSG, + fd, + msg, + 1, + 0, + 0, + callback, + ) else 0