dromedary-0.1.1/.cargo_vcs_info.json0000644000000001471046102023000130450ustar { "git": { "sha1": "37305d85fcffb07a7c95201d589d4f838ea868bf" }, "path_in_vcs": "dromedary" }dromedary-0.1.1/Cargo.lock0000644000000707131046102023000110260ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "aho-corasick" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "cfg-expr" version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d4ba6e40bd1184518716a6e1a781bf9160e286d219ccdb8ab2612e74cfe4789" dependencies = [ "smallvec", "target-lexicon 0.12.16", ] [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "displaydoc" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "dromedary" version = "0.1.1" dependencies = [ "gio", "glib", "lazy_static", "libc", "log", "nix", "path-clean", "percent-encoding", "pyo3", "pyo3-filelike", "regex", "tempfile", "url", "walkdir", ] [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", "windows-sys", ] [[package]] name = "fastrand" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] [[package]] name = "futures-channel" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", ] [[package]] name = "futures-core" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", "futures-util", ] [[package]] name = "futures-io" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "futures-task" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", "futures-macro", "futures-task", "pin-project-lite", "slab", ] [[package]] name = "getrandom" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", "r-efi", "wasip2", "wasip3", ] [[package]] name = "gio" version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "401b600a9795c46ff45890146968b712c96ce4e9393798804133e137bd81d89c" dependencies = [ "futures-channel", "futures-core", "futures-io", "futures-util", "gio-sys", "glib", "libc", "pin-project-lite", "smallvec", ] [[package]] name = "gio-sys" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64729ba2772c080448f9f966dba8f4456beeb100d8c28a865ef8a0f2ef4987e1" dependencies = [ "glib-sys", "gobject-sys", "libc", "system-deps", "windows-sys", ] [[package]] name = "glib" version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1b7df55594e0e787d1560e23f7e12d7360d0b22e7b7c228ec2488b9e59b1b6b" dependencies = [ "bitflags", "futures-channel", "futures-core", "futures-executor", "futures-task", "futures-util", "gio-sys", "glib-macros", "glib-sys", "gobject-sys", "libc", "memchr", "smallvec", ] [[package]] name = "glib-macros" version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bda575994e3689b1bc12f89c3df621ead46ff292623b76b4710a3a5b79be54bb" dependencies = [ "heck", "proc-macro2", "quote", "syn", ] [[package]] name = "glib-sys" version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eb23a616a3dbc7fc15bbd26f58756ff0b04c8a894df3f0680cd21011db6a642" dependencies = [ "libc", "system-deps", ] [[package]] name = "gobject-sys" version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18eda93f09d3778f38255b231b17ef67195013a592c91624a4daf8bead875565" dependencies = [ "glib-sys", "libc", "system-deps", ] [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "foldhash", ] [[package]] name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "icu_collections" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", "utf8_iter", "yoke", "zerofrom", "zerovec", ] [[package]] name = "icu_locale_core" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", "tinystr", "writeable", "zerovec", ] [[package]] name = "icu_normalizer" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", "zerovec", ] [[package]] name = "icu_normalizer_data" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", "writeable", "yoke", "zerofrom", "zerotrie", "zerovec", ] [[package]] name = "id-arena" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" [[package]] name = "idna" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", "utf8_iter", ] [[package]] name = "idna_adapter" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", ] [[package]] name = "indexmap" version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.0", "serde", "serde_core", ] [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "leb128fmt" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "nix" version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ "bitflags", "cfg-if", "cfg_aliases", "libc", ] [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "path-clean" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "potential_utf" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] [[package]] name = "prettyplease" version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", "syn", ] [[package]] name = "proc-macro2" version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" dependencies = [ "libc", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", ] [[package]] name = "pyo3-build-config" version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" dependencies = [ "target-lexicon 0.13.5", ] [[package]] name = "pyo3-ffi" version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" dependencies = [ "libc", "pyo3-build-config", ] [[package]] name = "pyo3-filelike" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a8cb6cd0231ea816b4452c0cd37b5215f9ec45b66ed3e748fad8eb39cfd4997" dependencies = [ "pyo3", ] [[package]] name = "pyo3-macros" version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", "syn", ] [[package]] name = "pyo3-macros-backend" version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", "syn", ] [[package]] name = "quote" version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "regex" version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", "regex-automata", "regex-syntax", ] [[package]] name = "regex-automata" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] [[package]] name = "regex-syntax" version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", "windows-sys", ] [[package]] name = "same-file" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ "winapi-util", ] [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", ] [[package]] name = "serde_core" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", "serde", "serde_core", "zmij", ] [[package]] name = "serde_spanned" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] [[package]] name = "slab" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "syn" version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "synstructure" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "system-deps" version = "7.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" dependencies = [ "cfg-expr", "heck", "pkg-config", "toml", "version-compare", ] [[package]] name = "target-lexicon" version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "target-lexicon" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "tempfile" version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom", "once_cell", "rustix", "windows-sys", ] [[package]] name = "tinystr" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", ] [[package]] name = "toml" version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" dependencies = [ "indexmap", "serde_core", "serde_spanned", "toml_datetime", "toml_parser", "toml_writer", "winnow", ] [[package]] name = "toml_datetime" version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] [[package]] name = "toml_writer" version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "url" version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", ] [[package]] name = "utf8_iter" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "version-compare" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" [[package]] name = "walkdir" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", ] [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ "wit-bindgen", ] [[package]] name = "wasip3" version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-encoder" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" dependencies = [ "leb128fmt", "wasmparser", ] [[package]] name = "wasm-metadata" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", "indexmap", "wasm-encoder", "wasmparser", ] [[package]] name = "wasmparser" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", "indexmap", "semver", ] [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ "windows-sys", ] [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-sys" version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] [[package]] name = "winnow" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" [[package]] name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" dependencies = [ "wit-bindgen-rust-macro", ] [[package]] name = "wit-bindgen-core" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", "heck", "wit-parser", ] [[package]] name = "wit-bindgen-rust" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", "indexmap", "prettyplease", "syn", "wasm-metadata", "wit-bindgen-core", "wit-component", ] [[package]] name = "wit-bindgen-rust-macro" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" dependencies = [ "anyhow", "prettyplease", "proc-macro2", "quote", "syn", "wit-bindgen-core", "wit-bindgen-rust", ] [[package]] name = "wit-component" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", "indexmap", "log", "serde", "serde_derive", "serde_json", "wasm-encoder", "wasm-metadata", "wasmparser", "wit-parser", ] [[package]] name = "wit-parser" version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", "indexmap", "log", "semver", "serde", "serde_derive", "serde_json", "unicode-xid", "wasmparser", ] [[package]] name = "writeable" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yoke" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", "zerofrom", ] [[package]] name = "yoke-derive" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", "syn", "synstructure", ] [[package]] name = "zerofrom" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", "syn", "synstructure", ] [[package]] name = "zerotrie" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", "zerofrom", ] [[package]] name = "zerovec" version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", "zerovec-derive", ] [[package]] name = "zerovec-derive" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" dromedary-0.1.1/Cargo.toml0000644000000034451046102023000110470ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2018" name = "dromedary" version = "0.1.1" authors = [ "Martin Packman ", "Jelmer Vernooij ", ] build = false autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "Transport layer for Breezy" homepage = "https://www.breezy-vcs.org/" documentation = "https://www.breezy-vcs.org/doc/" readme = false license = "GPL-2.0+" repository = "https://github.com/breezy-team/dromedary" resolver = "2" [features] default = ["pyo3"] gio = [ "dep:gio", "dep:glib", ] pyo3 = [ "dep:pyo3", "dep:pyo3-filelike", ] [lib] name = "dromedary" path = "src/lib.rs" [dependencies.gio] version = "0.22" optional = true [dependencies.glib] version = "0.22" optional = true [dependencies.lazy_static] version = "1" [dependencies.libc] version = "0.2" [dependencies.log] version = "0.4" [dependencies.path-clean] version = "1" [dependencies.percent-encoding] version = "2.1.0" [dependencies.pyo3] version = "0.28" optional = true [dependencies.pyo3-filelike] version = "0.5.2" optional = true [dependencies.regex] version = "1.5.4" [dependencies.tempfile] version = "3" [dependencies.url] version = "2" [dependencies.walkdir] version = "2.3" [target."cfg(unix)".dependencies.nix] version = ">=0.26" features = [ "fs", "uio", ] dromedary-0.1.1/Cargo.toml.orig000064400000000000000000000016751046102023000145110ustar 00000000000000[package] name = "dromedary" version = { workspace = true } edition = "2018" description = "Transport layer for Breezy" license = "GPL-2.0+" repository = "https://github.com/breezy-team/dromedary" homepage = "https://www.breezy-vcs.org/" documentation = "https://www.breezy-vcs.org/doc/" authors = [ "Martin Packman ", "Jelmer Vernooij "] [lib] [dependencies] url = { workspace = true } tempfile = "3" pyo3 = { workspace = true, optional = true } pyo3-filelike = { workspace = true, optional = true } path-clean = "1" walkdir = "2.3" lazy_static = "1" log = { workspace = true } libc = "0.2" regex = "1.5.4" percent-encoding = "2.1.0" gio = { version = "0.22", optional = true } glib = { version = "0.22", optional = true } [features] default = ["pyo3"] pyo3 = ["dep:pyo3", "dep:pyo3-filelike"] gio = ["dep:gio", "dep:glib"] [target.'cfg(unix)'.dependencies] nix = { workspace = true, features = ["fs", "uio"] } dromedary-0.1.1/__init__.py000064400000000000000000001773701046102023000137410ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport is an abstraction layer to handle file access. The abstraction is to allow access from the local filesystem, as well as remote (such as http or sftp). Transports are constructed from a string, being a URL or (as a degenerate case) a local filesystem path. This is typically the top directory of a bzrdir, repository, or similar object we are interested in working with. The Transport returned has methods to read, write and manipulate files within it. """ import contextlib import errno import logging import os import sys from collections.abc import Callable from io import BytesIO from stat import S_ISDIR from typing import Any, TypeVar from catalogus import registry from . import _hooks, _ui, errors, osutils, urlutils # Set up logging logger = logging.getLogger("dromedary") # Import the Rust extension try: from . import _transport_rs except ImportError: _transport_rs = None # a dictionary of open file streams. Keys are absolute paths, values are # transport defined. _file_streams: dict[str, Any] = {} def _get_protocol_handlers(): """Return a dictionary of {urlprefix: [factory]}.""" return transport_list_registry def _set_protocol_handlers(new_handlers): """Replace the current protocol handlers dictionary. WARNING this will remove all build in protocols. Use with care. """ global transport_list_registry transport_list_registry = new_handlers def _clear_protocol_handlers(): global transport_list_registry transport_list_registry = TransportListRegistry() def _get_transport_modules(): """Return a list of the modules providing transports.""" modules = set() for _prefix, factory_list in transport_list_registry.items(): for factory in factory_list: modules.add(factory.get_module()) # Add chroot and pathfilter directly, because there is no handler # registered for it. modules.add("dromedary.chroot") modules.add("dromedary.pathfilter") result = sorted(modules) return result class TransportListRegistry(registry.Registry): """A registry which simplifies tracking available Transports. A registration of a new protocol requires two steps: 1) register the prefix with the function register_transport( ) 2) register the protocol provider with the function register_transport_provider( ) ( and the "lazy" variant ) This is needed because: a) a single provider can support multiple protocols (like the ftp provider which supports both the ftp:// and the aftp:// protocols) b) a single protocol can have multiple providers (like the http:// protocol which was supported by both the urllib and pycurl providers) """ def register_transport_provider(self, key, obj): """Register a transport provider object for a protocol. Args: key: Protocol prefix (e.g., 'http://'). obj: Transport class or factory object. """ self.get(key).insert(0, registry._ObjectGetter(obj)) def register_lazy_transport_provider(self, key, module_name, member_name): """Register a transport provider with lazy loading. Args: key: Protocol prefix (e.g., 'http://'). module_name: Name of the module containing the transport class. member_name: Name of the transport class within the module. """ self.get(key).insert(0, registry._LazyObjectGetter(module_name, member_name)) def register_transport(self, key, help=None): """Register a transport protocol. Args: key: Protocol prefix (e.g., 'http://'). help: Optional help text describing the transport. """ self.register(key, [], help) transport_list_registry = TransportListRegistry() def register_transport_proto(prefix, help=None, info=None, register_netloc=False): """Register a transport protocol prefix. Args: prefix: Protocol prefix (e.g., 'http://'). help: Optional help text. info: Additional protocol information (unused). register_netloc: Whether to register for URL parsing with netloc. """ transport_list_registry.register_transport(prefix, help) if register_netloc: if not prefix.endswith("://"): raise ValueError(prefix) register_urlparse_netloc_protocol(prefix[:-3]) def register_lazy_transport(prefix, module, classname): """Register a transport with lazy class loading. Args: prefix: Protocol prefix (e.g., 'http://'). module: Module name containing the transport class. classname: Name of the transport class. """ if prefix not in transport_list_registry: register_transport_proto(prefix) transport_list_registry.register_lazy_transport_provider(prefix, module, classname) def register_transport(prefix, klass): """Register a transport class for a protocol prefix. Args: prefix: Protocol prefix (e.g., 'http://'). klass: Transport class to register. """ if prefix not in transport_list_registry: register_transport_proto(prefix) transport_list_registry.register_transport_provider(prefix, klass) def register_urlparse_netloc_protocol(protocol): """Ensure that protocol is setup to be used with urlparse netloc parsing.""" if protocol not in urlutils.urlparse.uses_netloc: urlutils.urlparse.uses_netloc.append(protocol) def _unregister_urlparse_netloc_protocol(protocol): """Remove protocol from urlparse netloc parsing. Except for tests, you should never use that function. Using it with 'http', for example, will break all http transports. """ if protocol in urlutils.urlparse.uses_netloc: urlutils.urlparse.uses_netloc.remove(protocol) def unregister_transport(scheme, factory): """Unregister a transport.""" l = transport_list_registry.get(scheme) for i in l: o = i.get_obj() if o == factory: transport_list_registry.get(scheme).remove(i) break if len(l) == 0: transport_list_registry.remove(scheme) class _CoalescedOffset: """A data container for keeping track of coalesced offsets.""" __slots__ = ["length", "ranges", "start"] def __init__(self, start, length, ranges): self.start = start self.length = length self.ranges = ranges def __lt__(self, other): return (self.start, self.length, self.ranges) < ( other.start, other.length, other.ranges, ) def __eq__(self, other): return (self.start, self.length, self.ranges) == ( other.start, other.length, other.ranges, ) def __repr__(self): return "{}({!r}, {!r}, {!r})".format( self.__class__.__name__, self.start, self.length, self.ranges ) class LateReadError: """A helper for transports which pretends to be a readable file. When read() is called, errors.ReadError is raised. """ def __init__(self, path): """Initialize LateReadError. Args: path: Path that will trigger the read error. """ self._path = path def close(self): """A no-op - do nothing.""" def __enter__(self): """Context manager entry.""" return self def __exit__(self, exc_type, exc_val, exc_tb): """Context manager exit.""" # If there was an error raised, prefer the original one try: self.close() except BaseException: if exc_type is None: raise return False def _fail(self): """Raise ReadError.""" raise errors.ReadError(self._path) def __iter__(self): """Iterator protocol - raises ReadError.""" self._fail() def read(self, count=-1): """Read method - raises ReadError. Args: count: Number of bytes to read (ignored). """ self._fail() def readlines(self): """Read lines method - raises ReadError.""" self._fail() class FileStream: """Base class for FileStreams.""" def __init__(self, transport, relpath): """Create a FileStream for relpath on transport.""" self.transport = transport self.relpath = relpath def _close(self): """A hook point for subclasses that need to take action on close.""" def __enter__(self): """Context manager entry.""" return self def __exit__(self, exc_type, exc_value, exc_tb): """Context manager exit.""" self.close() return False def close(self, want_fdatasync=False): """Close the file stream. Args: want_fdatasync: Whether to force data synchronization. """ if want_fdatasync: with contextlib.suppress(errors.TransportNotPossible): self.fdatasync() self._close() del _file_streams[self.transport.abspath(self.relpath)] def fdatasync(self): """Force data out to physical disk if possible. :raises errors.TransportNotPossible: If this transport has no way to flush to disk. """ raise errors.TransportNotPossible(f"{self.transport} cannot fdatasync") class FileFileStream(FileStream): """A file stream object returned by open_write_stream. This version uses a file like object to perform writes. """ def __init__(self, transport, relpath, file_handle): """Initialize FileFileStream. Args: transport: Transport instance. relpath: Relative path to the file. file_handle: File-like object for operations. """ FileStream.__init__(self, transport, relpath) self.file_handle = file_handle def _close(self): self.file_handle.close() def fdatasync(self): """Force data out to physical disk if possible.""" self.file_handle.flush() try: fileno = self.file_handle.fileno() except AttributeError as err: raise errors.TransportNotPossible() from err osutils.fdatasync(fileno) def write(self, bytes): """Write bytes to the file. Args: bytes: Data to write. Returns: Number of bytes written. """ class F: def __init__(self, f): self.f = f def write(self, b): self.f.write(b) return len(b) osutils.pump_string_file(bytes, F(self.file_handle)) return len(bytes) def flush(self): """Flush any buffered data.""" self.file_handle.flush() class AppendBasedFileStream(FileStream): """A file stream object returned by open_write_stream. This version uses append on a transport to perform writes. """ def write(self, bytes): """Write bytes by appending to the file. Args: bytes: Data to write. Returns: Number of bytes written. """ self.transport.append_bytes(self.relpath, bytes) return len(bytes) def flush(self): """Flush any buffered data (no-op for append-based streams).""" pass class TransportHooks(_hooks.Hooks): """Mapping of hook names to registered callbacks for transport hooks.""" def __init__(self): """Initialize TransportHooks.""" super().__init__() self.add_hook( "post_connect", "Called after a new connection is established or a reconnect " "occurs. The sole argument passed is either the connected " "transport or smart medium instance.", (2, 5), ) class Transport: """This class encapsulates methods for retrieving or putting a file from/to a storage location. :ivar base: Base URL for the transport; should always end in a slash. """ # implementations can override this if it is more efficient # for them to combine larger read chunks together _max_readv_combine = 50 # It is better to read this much more data in order, rather # than doing another seek. Even for the local filesystem, # there is a benefit in just reading. # TODO: jam 20060714 Do some real benchmarking to figure out # where the biggest benefit between combining reads and # and seeking is. Consider a runtime auto-tune. _bytes_to_read_before_seek = 0 hooks = TransportHooks() base: str def __init__(self, base): """Initialize a Transport. Args: base: Base URL for the transport; should always end in a slash. """ super().__init__() self.base = base (self._raw_base, self._segment_parameters) = urlutils.split_segment_parameters( base ) def _translate_error(self, e, path, raise_generic=True): """Translate an IOError or OSError into an appropriate bzr error. This handles things like ENOENT, ENOTDIR, EEXIST, and EACCESS """ if getattr(e, "errno", None) is not None: if e.errno in (errno.ENOENT, errno.ENOTDIR): raise errors.NoSuchFile(path, extra=e) elif e.errno == errno.EINVAL: logger.debug("EINVAL returned on path %s: %r", path, e) raise errors.NoSuchFile(path, extra=e) # I would rather use errno.EFOO, but there doesn't seem to be # any matching for 267 # This is the error when doing a listdir on a file: # WindowsError: [Errno 267] The directory name is invalid if sys.platform == "win32" and e.errno in (errno.ESRCH, 267): raise errors.NoSuchFile(path, extra=e) if e.errno == errno.EEXIST: raise errors.FileExists(path, extra=e) if e.errno == errno.EACCES: raise errors.PermissionDenied(path, extra=e) if e.errno == errno.ENOTEMPTY: raise errors.DirectoryNotEmpty(path, extra=e) if e.errno == errno.EBUSY: raise errors.ResourceBusy(path, extra=e) if isinstance(e, (NotADirectoryError, FileNotFoundError)): raise errors.NoSuchFile(path, extra=e) if raise_generic: raise errors.TransportError(msg="Transport operation failed", orig_error=e) def clone(self, offset=None): """Return a new Transport object, cloned from the current location, using a subdirectory or parent directory. This allows connections to be pooled, rather than a new one needed for each subdir. """ raise NotImplementedError(self.clone) def create_prefix(self, mode=None): """Create all the directories leading down to self.base.""" cur_transport = self needed = [cur_transport] # Recurse upwards until we can create a directory successfully while True: new_transport = cur_transport.clone("..") if new_transport.base == cur_transport.base: raise errors.TransportError( f"Failed to create path prefix for {cur_transport.base}." ) try: new_transport.mkdir(".", mode=mode) except errors.NoSuchFile: needed.append(new_transport) cur_transport = new_transport except errors.FileExists: break else: break # Now we only need to create child directories while needed: cur_transport = needed.pop() cur_transport.ensure_base(mode=mode) def ensure_base(self, mode=None): """Ensure that the directory this transport references exists. This will create a directory if it doesn't exist. :return: True if the directory was created, False otherwise. """ # The default implementation just uses "Easier to ask for forgiveness # than permission". We attempt to create the directory, and just # suppress FileExists and PermissionDenied (for Windows) exceptions. try: self.mkdir(".", mode=mode) except (errors.FileExists, errors.PermissionDenied): return False except errors.TransportNotPossible: if self.has("."): return False raise else: return True def external_url(self): """Return a URL for self that can be given to an external process. There is no guarantee that the URL can be accessed from a different machine - e.g. file:/// urls are only usable on the local machine, sftp:/// urls when the server is only bound to localhost are only usable from localhost etc. NOTE: This method may remove security wrappers (e.g. on chroot transports) and thus should *only* be used when the result will not be used to obtain a new transport within breezy. Ideally chroot transports would know enough to cause the external url to be the exact one used that caused the chrooting in the first place, but that is not currently the case. :return: A URL that can be given to another process. :raises InProcessTransport: If the transport is one that cannot be accessed out of the current process (e.g. a MemoryTransport) then InProcessTransport is raised. """ raise NotImplementedError(self.external_url) def get_segment_parameters(self): """Return the segment parameters for the top segment of the URL.""" return self._segment_parameters def set_segment_parameter(self, name, value): """Set a segment parameter. Args: name: Segment parameter name (urlencoded string) value: Segment parameter value (urlencoded string) """ if value is None: with contextlib.suppress(KeyError): del self._segment_parameters[name] else: self._segment_parameters[name] = value self.base = urlutils.join_segment_parameters( self._raw_base, self._segment_parameters ) def _pump(self, from_file, to_file): """Most children will need to copy from one file-like object or string to another one. This just gives them something easy to call. """ return osutils.pumpfile(from_file, to_file) def _get_total(self, multi): """Try to figure out how many entries are in multi, but if not possible, return None. """ try: return len(multi) except TypeError: # We can't tell how many, because relpaths is a generator return None def _report_activity(self, bytes, direction): """Notify that this transport has activity. Implementations should call this from all methods that actually do IO. Be careful that it's not called twice, if one method is implemented on top of another. Args: bytes: Number of bytes read or written. direction: 'read' or 'write' or None. """ _ui.report_transport_activity(self, bytes, direction) def _update_pb(self, pb, msg, count, total): """Update the progress bar based on the current count and total available, total may be None if it was not possible to determine. """ if pb is None: return if total is None: pb.update(msg, count, count + 1) else: pb.update(msg, count, total) def _iterate_over(self, multi, func, pb, msg, expand=True): """Iterate over all entries in multi, passing them to func, and update the progress bar as you go along. :param expand: If True, the entries will be passed to the function by expanding the tuple. If False, it will be passed as a single parameter. """ total = self._get_total(multi) result = [] for count, entry in enumerate(multi): self._update_pb(pb, msg, count, total) if expand: result.append(func(*entry)) else: result.append(func(entry)) return tuple(result) def abspath(self, relpath): """Return the full url to the given relative path. :param relpath: a string of a relative path """ # XXX: Robert Collins 20051016 - is this really needed in the public # interface ? raise NotImplementedError(self.abspath) def recommended_page_size(self): """Return the recommended page size for this transport. This is potentially different for every path in a given namespace. For example, local transports might use an operating system call to get the block size for a given path, which can vary due to mount points. :return: The page size in bytes. """ return 4 * 1024 def relpath(self, abspath): """Return the local path portion from a given absolute path. This default implementation is not suitable for filesystems with aliasing, such as that given by symlinks, where a path may not start with our base, but still be a relpath once aliasing is resolved. """ # TODO: This might want to use dromedary.osutils.relpath # but we have to watch out because of the prefix issues if not (abspath == self.base[:-1] or abspath.startswith(self.base)): raise errors.PathNotChild(abspath, self.base) pl = len(self.base) return abspath[pl:].strip("/") def local_abspath(self, relpath): """Return the absolute path on the local filesystem. This function will only be defined for Transports which have a physical local filesystem representation. :raises errors.NotLocalUrl: When no local path representation is available. """ raise errors.NotLocalUrl(self.abspath(relpath)) def has(self, relpath): """Does the file relpath exist? Note that some transports MAY allow querying on directories, but this is not part of the protocol. In other words, the results of t.has("a_directory_name") are undefined. :rtype: bool """ raise NotImplementedError(self.has) def has_any(self, relpaths): """Return True if any of the paths exist.""" return any(self.has(relpath) for relpath in relpaths) def iter_files_recursive(self): """Iter the relative paths of files in the transports sub-tree. *NOTE*: This only lists *files*, not subdirectories! As with other listing functions, only some transports implement this,. you may check via listable() to determine if it will. """ raise errors.TransportNotPossible( "This transport has not " "implemented iter_files_recursive " "(but must claim to be listable " "to trigger this error)." ) def get(self, relpath): """Get the file at the given relative path. This may fail in a number of ways: - HTTP servers may return content for a directory. (unexpected content failure) - FTP servers may indicate errors.NoSuchFile for a directory. - SFTP servers may give a file handle for a directory that will fail on read(). For correct use of the interface, be sure to catch PathError when calling it and catch errors.ReadError when reading from the returned object. :param relpath: The relative path to the file :rtype: File-like object. """ raise NotImplementedError(self.get) def get_bytes(self, relpath): """Get a raw string of the bytes for a file at the given location. :param relpath: The relative path to the file """ f = self.get(relpath) try: return f.read() finally: f.close() def get_smart_medium(self): """Return a smart client medium for this transport if possible. A smart medium doesn't imply the presence of a smart server: it implies that the smart protocol can be tunnelled via this transport. :raises errors.NoSmartMedium: if no smart server medium is available. """ raise errors.NoSmartMedium(self) def readv(self, relpath, offsets, adjust_for_latency=False, upper_limit=None): """Get parts of the file at the given relative path. Args: relpath: The path to read data from. offsets: A list of (offset, size) tuples. adjust_for_latency: Adjust the requested offsets to accomodate transport latency. This may re-order the offsets, expand them to grab adjacent data when there is likely a high cost to requesting data relative to delivering it. upper_limit: When adjust_for_latency is True setting upper_limit allows the caller to tell the transport about the length of the file, so that requests are not issued for ranges beyond the end of the file. This matters because some servers and/or transports error in such a case rather than just satisfying the available ranges. upper_limit should always be provided when adjust_for_latency is True, and should be the size of the file in bytes. Returns: A list or generator of (offset, data) tuples """ if adjust_for_latency: # Design note: We may wish to have different algorithms for the # expansion of the offsets per-transport. E.g. for local disk to # use page-aligned expansion. If that is the case consider the # following structure: # - a test that transport.readv uses self._offset_expander or some # similar attribute, to do the expansion # - a test for each transport that it has some known-good offset # expander # - unit tests for each offset expander # - a set of tests for the offset expander interface, giving # baseline behaviour (which the current transport # adjust_for_latency tests could be repurposed to). offsets = self._sort_expand_and_combine(offsets, upper_limit) return self._readv(relpath, offsets) def _readv(self, relpath, offsets): """Get parts of the file at the given relative path. :param relpath: The path to read. :param offsets: A list of (offset, size) tuples. :return: A list or generator of (offset, data) tuples """ if not offsets: return fp = self.get(relpath) return self._seek_and_read(fp, offsets, relpath) def _seek_and_read(self, fp, offsets, relpath=""): """An implementation of readv that uses fp.seek and fp.read. This uses _coalesce_offsets to issue larger reads and fewer seeks. :param fp: A file-like object that supports seek() and read(size). Note that implementations are allowed to call .close() on this file handle, so don't trust that you can use it for other work. :param offsets: A list of offsets to be read from the given file. :return: yield (pos, data) tuples for each request """ try: yield from _transport_rs.seek_and_read( fp, offsets, max_readv_combine=self._max_readv_combine, bytes_to_read_before_seek=self._bytes_to_read_before_seek, path=relpath, ) finally: fp.close() def _sort_expand_and_combine(self, offsets, upper_limit): """Helper for readv. :param offsets: A readv vector - (offset, length) tuples. :param upper_limit: The highest byte offset that may be requested. :return: A readv vector that will read all the regions requested by offsets, in start-to-end order, with no duplicated regions, expanded by the transports recommended page size. """ return _transport_rs.sort_expand_and_combine( offsets, upper_limit, self.recommended_page_size() ) @staticmethod def _coalesce_offsets(offsets, limit=None, fudge_factor=None, max_size=None): """Yield coalesced offsets. With a long list of neighboring requests, combine them into a single large request, while retaining the original offsets. Turns [(15, 10), (25, 10)] => [(15, 20, [(0, 10), (10, 10)])] Note that overlapping requests are not permitted. (So [(15, 10), (20, 10)] will raise a ValueError.) This is because the data we access never overlaps, and it allows callers to trust that we only need any byte of data for 1 request (so nothing needs to be buffered to fulfill a second request.) :param offsets: A list of (start, length) pairs :param limit: Only combine a maximum of this many pairs Some transports penalize multiple reads more than others, and sometimes it is better to return early. 0 means no limit :param fudge_factor: All transports have some level of 'it is better to read some more data and throw it away rather than seek', so collapse if we are 'close enough' :param max_size: Create coalesced offsets no bigger than this size. When a single offset is bigger than 'max_size', it will keep its size and be alone in the coalesced offset. 0 means no maximum size. :return: return a list of _CoalescedOffset objects, which have members for where to start, how much to read, and how to split those chunks back up """ return [ _CoalescedOffset(start, length, ranges) for start, length, ranges in _transport_rs.coalesce_offsets( offsets, limit, fudge_factor, max_size ) ] def put_bytes(self, relpath: str, raw_bytes: bytes, mode=None): """Atomically put the supplied bytes into the given location. :param relpath: The location to put the contents, relative to the transport base. :param raw_bytes: A bytestring of data. :param mode: Create the file with the given mode. :return: None """ if not isinstance(raw_bytes, bytes): raise TypeError(f"raw_bytes must be a plain string, not {type(raw_bytes)}") return self.put_file(relpath, BytesIO(raw_bytes), mode=mode) def put_bytes_non_atomic( self, relpath, raw_bytes: bytes, mode=None, create_parent_dir=False, dir_mode=None, ): """Copy the string into the target location. This function is not strictly safe to use. See Transport.put_bytes_non_atomic for more information. :param relpath: The remote location to put the contents. :param raw_bytes: A string object containing the raw bytes to write into the target file. :param mode: Possible access permissions for new file. None means do not set remote permissions. :param create_parent_dir: If we cannot create the target file because the parent directory does not exist, go ahead and create it, and then try again. :param dir_mode: Possible access permissions for new directories. """ if not isinstance(raw_bytes, bytes): raise TypeError(f"raw_bytes must be a plain string, not {type(raw_bytes)}") self.put_file_non_atomic( relpath, BytesIO(raw_bytes), mode=mode, create_parent_dir=create_parent_dir, dir_mode=dir_mode, ) def put_file(self, relpath, f, mode=None): """Copy the file-like object into the location. :param relpath: Location to put the contents, relative to base. :param f: File-like object. :param mode: The mode for the newly created file, None means just use the default. :return: The length of the file that was written. """ raise NotImplementedError(self.put_file) def put_file_non_atomic( self, relpath, f, mode=None, create_parent_dir=False, dir_mode=None ): """Copy the file-like object into the target location. This function is not strictly safe to use. It is only meant to be used when you already know that the target does not exist. It is not safe, because it will open and truncate the remote file. So there may be a time when the file has invalid contents. :param relpath: The remote location to put the contents. :param f: File-like object. :param mode: Possible access permissions for new file. None means do not set remote permissions. :param create_parent_dir: If we cannot create the target file because the parent directory does not exist, go ahead and create it, and then try again. :param dir_mode: Possible access permissions for new directories. """ # Default implementation just does an atomic put. try: return self.put_file(relpath, f, mode=mode) except errors.NoSuchFile: if not create_parent_dir: raise parent_dir = os.path.dirname(relpath) if parent_dir: self.mkdir(parent_dir, mode=dir_mode) return self.put_file(relpath, f, mode=mode) def mkdir(self, relpath, mode=None): """Create a directory at the given path.""" raise NotImplementedError(self.mkdir) def open_write_stream(self, relpath, mode=None): """Open a writable file stream at relpath. A file stream is a file like object with a write() method that accepts bytes to write.. Buffering may occur internally until the stream is closed with stream.close(). Calls to readv or the get_* methods will be synchronised with any internal buffering that may be present. :param relpath: The relative path to the file. :param mode: The mode for the newly created file, None means just use the default :return: A FileStream. FileStream objects have two methods, write() and close(). There is no guarantee that data is committed to the file if close() has not been called (even if get() is called on the same path). """ raise NotImplementedError(self.open_write_stream) def append_file(self, relpath, f, mode=None): """Append bytes from a file-like object to a file at relpath. The file is created if it does not already exist. :param f: a file-like object of the bytes to append. :param mode: Unix mode for newly created files. This is not used for existing files. :returns: the length of relpath before the content was written to it. """ raise NotImplementedError(self.append_file) def append_bytes(self, relpath, data, mode=None): """Append bytes to a file at relpath. The file is created if it does not already exist. :param relpath: The relative path to the file. :param data: a string of the bytes to append. :param mode: Unix mode for newly created files. This is not used for existing files. :returns: the length of relpath before the content was written to it. """ if not isinstance(data, bytes): raise TypeError(f"bytes must be a plain string, not {type(data)}") return self.append_file(relpath, BytesIO(data), mode=mode) def copy(self, rel_from, rel_to): """Copy the item at rel_from to the location at rel_to. Override this for efficiency if a specific transport can do it faster than this default implementation. """ with self.get(rel_from) as f: self.put_file(rel_to, f) def copy_to(self, relpaths, other, mode=None, pb=None): """Copy a set of entries from self into another Transport. :param relpaths: A list/generator of entries to be copied. :param mode: This is the target mode for the newly created files TODO: This interface needs to be updated so that the target location can be different from the source location. """ # The dummy implementation just does a simple get + put def copy_entry(path): other.put_file(path, self.get(path), mode=mode) return len( self._iterate_over(relpaths, copy_entry, pb, "copy_to", expand=False) ) def copy_tree(self, from_relpath, to_relpath): """Copy a subtree from one relpath to another. If a faster implementation is available, specific transports should implement it. """ source = self.clone(from_relpath) target = self.clone(to_relpath) # create target directory with the same rwx bits as source. # use mask to ensure that bits other than rwx are ignored. stat = self.stat(from_relpath) target.mkdir(".", stat.st_mode & 0o777) source.copy_tree_to_transport(target) def copy_tree_to_transport(self, to_transport): """Copy a subtree from one transport to another. self.base is used as the source tree root, and to_transport.base is used as the target. to_transport.base must exist (and be a directory). """ files = [] directories = ["."] while directories: dir = directories.pop() if dir != ".": to_transport.mkdir(dir) for path in self.list_dir(dir): path = dir + "/" + path stat = self.stat(path) if S_ISDIR(stat.st_mode): directories.append(path) else: files.append(path) self.copy_to(files, to_transport) def rename(self, rel_from, rel_to): """Rename a file or directory. This *must* fail if the destination is a nonempty directory - it must not automatically remove it. It should raise errors.DirectoryNotEmpty, or some other errors.PathError if the case can't be specifically detected. If the destination is an empty directory or a file this function may either fail or succeed, depending on the underlying transport. It should not attempt to remove the destination if overwriting is not the native transport behaviour. If at all possible the transport should ensure that the rename either completes or not, without leaving the destination deleted and the new file not moved in place. This is intended mainly for use in implementing LockDir. """ # transports may need to override this raise NotImplementedError(self.rename) def move(self, rel_from, rel_to): """Move the item at rel_from to the location at rel_to. The destination is deleted if possible, even if it's a non-empty directory tree. If a transport can directly implement this it is suggested that it do so for efficiency. """ if S_ISDIR(self.stat(rel_from).st_mode): self.copy_tree(rel_from, rel_to) self.delete_tree(rel_from) else: self.copy(rel_from, rel_to) self.delete(rel_from) def delete(self, relpath): """Delete the item at relpath.""" raise NotImplementedError(self.delete) def delete_tree(self, relpath): """Delete an entire tree. This may require a listable transport.""" subtree = self.clone(relpath) files = [] directories = ["."] pending_rmdirs = [] while directories: dir = directories.pop() if dir != ".": pending_rmdirs.append(dir) for path in subtree.list_dir(dir): path = dir + "/" + path stat = subtree.stat(path) if S_ISDIR(stat.st_mode): directories.append(path) else: files.append(path) for file in files: subtree.delete(file) pending_rmdirs.reverse() for dir in pending_rmdirs: subtree.rmdir(dir) self.rmdir(relpath) def __repr__(self): """Return string representation of the transport.""" return f"<{self.__module__}.{self.__class__.__name__} url={self.base}>" def stat(self, relpath): """Return the stat information for a file. WARNING: This may not be implementable for all protocols, so use sparingly. NOTE: This returns an object with fields such as 'st_size'. It MAY or MAY NOT return the literal result of an os.stat() call, so all access should be via named fields. ALSO NOTE: Stats of directories may not be supported on some transports. """ raise NotImplementedError(self.stat) def rmdir(self, relpath): """Remove a directory at the given path.""" raise NotImplementedError def readlink(self, relpath): """Return a string representing the path to which the symbolic link points.""" raise errors.TransportNotPossible( f"Dereferencing symlinks is not supported on {self}" ) def hardlink(self, source, link_name): """Create a hardlink pointing to source named link_name.""" raise errors.TransportNotPossible(f"Hard links are not supported on {self}") def symlink(self, source, link_name): """Create a symlink pointing to source named link_name.""" raise errors.TransportNotPossible(f"Symlinks are not supported on {self}") def listable(self): """Return True if this store supports listing.""" raise NotImplementedError(self.listable) def list_dir(self, relpath): """Return a list of all files at the given location. WARNING: many transports do not support this, so trying avoid using it if at all possible. """ raise errors.TransportNotPossible( "Transport {!r} has not " "implemented list_dir " "(but must claim to be listable " "to trigger this error).".format(self) ) def lock_read(self, relpath): """Lock the given file for shared (read) access. WARNING: many transports do not support this, so trying avoid using it. These methods may be removed in the future. Transports may raise errors.TransportNotPossible if OS-level locks cannot be taken over this transport. :return: A lock object, which should contain an unlock() function. """ raise errors.TransportNotPossible(f"transport locks not supported on {self}") def lock_write(self, relpath): """Lock the given file for exclusive (write) access. WARNING: many transports do not support this, so trying avoid using it. These methods may be removed in the future. Transports may raise errors.TransportNotPossible if OS-level locks cannot be taken over this transport. :return: A lock object, which should contain an unlock() function. """ raise errors.TransportNotPossible(f"transport locks not supported on {self}") def is_readonly(self): """Return true if this connection cannot be written to.""" return False def _can_roundtrip_unix_modebits(self): """Return true if this transport can store and retrieve unix modebits. (For example, 0700 to make a directory owner-private.) Note: most callers will not want to switch on this, but should rather just try and set permissions and let them be either stored or not. This is intended mainly for the use of the test suite. Warning: this is not guaranteed to be accurate as sometimes we can't be sure: for example with vfat mounted on unix, or a windows sftp server. """ # TODO: Perhaps return a e.g. TransportCharacteristics that can answer # several questions about the transport. return False def _reuse_for(self, other_base): # This is really needed for ConnectedTransport only, but it's easier to # have Transport refuses to be reused than testing that the reuse # should be asked to ConnectedTransport only. return None def disconnect(self): """Disconnect the transport. This is primarily for ConnectedTransport subclasses, but is implemented as a no-op in the base Transport class for convenience. """ # This is really needed for ConnectedTransport only, but it's easier to # have Transport do nothing than testing that the disconnect should be # asked to ConnectedTransport only. pass def _redirected_to(self, source, target): """Returns a transport suitable to re-issue a redirected request. :param source: The source url as returned by the server. :param target: The target url as returned by the server. The redirection can be handled only if the relpath involved is not renamed by the redirection. :returns: A transport :raise errors.UnusableRedirect: when redirection can not be provided """ # This returns None by default, meaning the transport can't handle the # redirection. raise errors.UnusableRedirect( source, target, "transport does not support redirection" ) class _SharedConnection: """A connection shared between several transports.""" def __init__(self, connection=None, credentials=None, base=None): """Constructor. :param connection: An opaque object specific to each transport. :param credentials: An opaque object containing the credentials used to create the connection. """ self.connection = connection self.credentials = credentials self.base = base class ConnectedTransport(Transport): """A transport connected to a remote server. This class provide the basis to implement transports that need to connect to a remote server. Host and credentials are available as private attributes, cloning preserves them and share the underlying, protocol specific, connection. """ def __init__(self, base, _from_transport=None): """Constructor. The caller should ensure that _from_transport points at the same host as the new base. :param base: transport root URL :param _from_transport: optional transport to build from. The built transport will share the connection with this transport. """ if not base.endswith("/"): base += "/" self._parsed_url = self._split_url(base) if _from_transport is not None: # Copy the password as it does not appear in base and will be lost # otherwise. It can appear in the _split_url above if the user # provided it on the command line. Otherwise, daughter classes will # prompt the user for one when appropriate. self._parsed_url.password = _from_transport._parsed_url.password self._parsed_url.quoted_password = ( _from_transport._parsed_url.quoted_password ) base = str(self._parsed_url) super().__init__(base) if _from_transport is None: self._shared_connection = _SharedConnection() else: self._shared_connection = _from_transport._shared_connection @property def _user(self): return self._parsed_url.user @property def _password(self): return self._parsed_url.password @property def _host(self): return self._parsed_url.host @property def _port(self): return self._parsed_url.port @property def _path(self): return self._parsed_url.path @property def _scheme(self): return self._parsed_url.scheme def clone(self, offset=None): """Return a new transport with root at self.base + offset. We leave the daughter classes take advantage of the hint that it's a cloning not a raw creation. """ if offset is None: return self.__class__(self.base, _from_transport=self) else: return self.__class__(self.abspath(offset), _from_transport=self) @staticmethod def _split_url(url): return urlutils.URL.from_string(url) @staticmethod def _unsplit_url(scheme, user, password, host, port, path): """Build the full URL for the given already URL encoded path. user, password, host and path will be quoted if they contain reserved chars. Args: scheme: protocol user: login password: associated password host: the server address port: the associated port path: the absolute path on the server :return: The corresponding URL. """ netloc = urlutils.quote(host) if user is not None: # Note that we don't put the password back even if we # have one so that it doesn't get accidentally # exposed. netloc = f"{urlutils.quote(user)}@{netloc}" if port is not None: netloc = "%s:%d" % (netloc, port) path = urlutils.escape(path) return urlutils.urlparse.urlunparse((scheme, netloc, path, None, None, None)) def relpath(self, abspath): """Return the local path portion from a given absolute path.""" parsed_url = self._split_url(abspath) error = [] if parsed_url.scheme != self._parsed_url.scheme: error.append("scheme mismatch") if parsed_url.user != self._parsed_url.user: error.append("user name mismatch") if parsed_url.host != self._parsed_url.host: error.append("host mismatch") if parsed_url.port != self._parsed_url.port: error.append("port mismatch") if not ( parsed_url.path == self._parsed_url.path[:-1] or parsed_url.path.startswith(self._parsed_url.path) ): error.append("path mismatch") if error: extra = ", ".join(error) raise errors.PathNotChild(abspath, self.base, extra=extra) pl = len(self._parsed_url.path) return parsed_url.path[pl:].strip("/") def abspath(self, relpath): """Return the full url to the given relative path. Args: relpath: the relative path urlencoded :returns: the Unicode version of the absolute path for relpath. """ return str(self._parsed_url.clone(relpath)) def _remote_path(self, relpath): """Return the absolute path part of the url to the given relative path. This is the path that the remote server expect to receive in the requests, daughter classes should redefine this method if needed and use the result to build their requests. Args: relpath: the path relative to the transport base urlencoded. :return: the absolute Unicode path on the server, """ return self._parsed_url.clone(relpath).path def _get_shared_connection(self): """Get the object shared amongst cloned transports. This should be used only by classes that needs to extend the sharing with objects other than transports. Use _get_connection to get the connection itself. """ return self._shared_connection def _set_connection(self, connection, credentials=None): """Record a newly created connection with its associated credentials. Note: To ensure that connection is still shared after a temporary failure and a new one needs to be created, daughter classes should always call this method to set the connection and do so each time a new connection is created. Args: connection: An opaque object representing the connection used by the daughter class. credentials: An opaque object representing the credentials needed to create the connection. """ self._shared_connection.connection = connection self._shared_connection.credentials = credentials for hook in self.hooks["post_connect"]: hook(self) def _get_connection(self): """Returns the transport specific connection object.""" return self._shared_connection.connection def _get_credentials(self): """Returns the credentials used to establish the connection.""" return self._shared_connection.credentials def _update_credentials(self, credentials): """Update the credentials of the current connection. Some protocols can renegociate the credentials within a connection, this method allows daughter classes to share updated credentials. :param credentials: the updated credentials. """ # We don't want to call _set_connection here as we are only updating # the credentials not creating a new connection. self._shared_connection.credentials = credentials def _reuse_for(self, other_base): """Returns a transport sharing the same connection if possible. Note: we share the connection if the expected credentials are the same: (host, port, user). Some protocols may disagree and redefine the criteria in daughter classes. Note: we don't compare the passwords here because other_base may have been obtained from an existing transport.base which do not mention the password. :param other_base: the URL we want to share the connection with. :return: A new transport or None if the connection cannot be shared. """ try: parsed_url = self._split_url(other_base) except urlutils.InvalidURL: # No hope in trying to reuse an existing transport for an invalid # URL return None transport = None # Don't compare passwords, they may be absent from other_base or from # self and they don't carry more information than user anyway. if ( parsed_url.scheme == self._parsed_url.scheme and parsed_url.user == self._parsed_url.user and parsed_url.host == self._parsed_url.host and parsed_url.port == self._parsed_url.port ): path = parsed_url.path if not path.endswith("/"): # This normally occurs at __init__ time, but it's easier to do # it now to avoid creating two transports for the same base. path += "/" if self._parsed_url.path == path: # shortcut, it's really the same transport return self # We don't call clone here because the intent is different: we # build a new transport on a different base (which may be totally # unrelated) but we share the connection. transport = self.__class__(other_base, _from_transport=self) return transport def disconnect(self): """Disconnect the transport. If and when required the transport willl reconnect automatically. """ raise NotImplementedError(self.disconnect) def get_transport_from_path(path, possible_transports=None): """Open a transport for a local path. :param path: Local path as byte or unicode string :return: Transport object for path """ return get_transport_from_url(urlutils.local_path_to_url(path), possible_transports) def get_transport_from_url(url, possible_transports=None): """Open a transport to access a URL. Args: base: a URL transports: optional reusable transports list. If not None, created transports will be added to the list. Returns: A new transport optionally sharing its connection with one of possible_transports. """ transport = None if possible_transports is not None: for t in possible_transports: try: t_same_connection = t._reuse_for(url) except AttributeError: continue if t_same_connection is not None: # Add only new transports if t_same_connection not in possible_transports: possible_transports.append(t_same_connection) return t_same_connection last_err = None for proto, factory_list in transport_list_registry.items(): if proto is not None and url.startswith(proto): transport, last_err = _try_transport_factories(url, factory_list) if transport: if possible_transports is not None: if transport in possible_transports: raise AssertionError() possible_transports.append(transport) return transport if not urlutils.is_url(url): raise urlutils.InvalidURL(path=url) raise errors.UnsupportedProtocol(url, last_err) def _try_transport_factories(base, factory_list): last_err = None for factory in factory_list: try: return factory.get_obj()(base), None except errors.DependencyNotPresent as e: logger.debug( "failed to instantiate transport %r for %r: %r", factory, base, e ) last_err = e continue return None, last_err T = TypeVar("T") def do_catching_redirections( action: Callable[[Transport], T], transport: Transport, redirected: Callable[[Transport, errors.RedirectRequested, str], Transport], ) -> T: """Execute an action with given transport catching redirections. This is a facility provided for callers needing to follow redirections silently. The silence is relative: it is the caller responsability to inform the user about each redirection or only inform the user of a user via the exception parameter. Args: action: A callable, what the caller want to do while catching redirections. transport: The initial transport used. redirected: A callable receiving the redirected transport and the errors.RedirectRequested exception. :return: Whatever 'action' returns """ MAX_REDIRECTIONS = 8 # If a loop occurs, there is little we can do. So we don't try to detect # them, just getting out if too much redirections occurs. The solution # is outside: where the loop is defined. for _redirections in range(MAX_REDIRECTIONS): try: return action(transport) except errors.RedirectRequested as e: redirection_notice = "{} is{} redirected to {}".format( e.source, e.permanently, e.target ) transport = redirected(transport, e, redirection_notice) else: # Loop exited without resolving redirect ? Either the # user has kept a very very very old reference or a loop # occurred in the redirections. Nothing we can cure here: # tell the user. Note that as the user has been informed # about each redirection (it is the caller responsibility # to do that in redirected via the provided # redirection_notice). The caller may provide more # information if needed (like what file or directory we # were trying to act upon when the redirection loop # occurred). raise errors.TooManyRedirections() class Server: """A Transport Server. The Server interface provides a server for a given transport type. """ def start_server(self): """Setup the server to service requests.""" def stop_server(self): """Remove the server and cleanup any resources it owns.""" def open_file(url): """Open a file from a URL. :param url: URL to open :return: A file-like object. """ base, filename = urlutils.split(url) transport = get_transport_from_url(base) return open_file_via_transport(filename, transport) def open_file_via_transport(filename, transport): """Open a file using the transport, follow redirects as necessary.""" def open_file(transport): return transport.get(filename) def follow_redirection(transport, e, redirection_notice): logger.debug("%s", redirection_notice) base, _filename = urlutils.split(e.target) redirected_transport = get_transport_from_url(base) return redirected_transport return do_catching_redirections(open_file, transport, follow_redirection) # None is the default transport, for things with no url scheme register_transport_proto( "file://", help="Access using the standard filesystem (default)" ) register_lazy_transport("file://", "dromedary.local", "LocalTransport") register_transport_proto( "sftp://", help="Access using SFTP (most SSH servers provide SFTP).", register_netloc=True, ) register_lazy_transport("sftp://", "dromedary.sftp", "SFTPTransport") # Decorated http transport register_transport_proto( "http+urllib://", # help="Read-only access of branches exported on the web." register_netloc=True, ) register_lazy_transport("http+urllib://", "dromedary.http.urllib", "HttpTransport") register_transport_proto( "https+urllib://", # help="Read-only access of branches exported on the web using SSL." register_netloc=True, ) register_lazy_transport("https+urllib://", "dromedary.http.urllib", "HttpTransport") # Default http transports (last declared wins (if it can be imported)) register_transport_proto( "http://", help="Read-only access of branches exported on the web." ) register_transport_proto( "https://", help="Read-only access of branches exported on the web using SSL." ) # The default http implementation is urllib register_lazy_transport("http://", "dromedary.http.urllib", "HttpTransport") register_lazy_transport("https://", "dromedary.http.urllib", "HttpTransport") register_transport_proto("gio+", help="Access using any GIO supported protocols.") register_lazy_transport("gio+", "dromedary.gio_transport", "GioTransport") register_transport_proto("memory://") register_lazy_transport("memory://", "dromedary.memory", "MemoryTransport") register_transport_proto( "readonly+", # help="This modifier converts any transport to be readonly." ) register_lazy_transport("readonly+", "dromedary.readonly", "ReadonlyTransportDecorator") register_transport_proto("fakenfs+") register_lazy_transport("fakenfs+", "dromedary.fakenfs", "FakeNFSTransportDecorator") register_transport_proto("log+") register_lazy_transport("log+", "dromedary.log", "TransportLogDecorator") register_transport_proto("trace+") register_lazy_transport("trace+", "dromedary.trace", "TransportTraceDecorator") register_transport_proto("unlistable+") register_lazy_transport( "unlistable+", "dromedary.unlistable", "UnlistableTransportDecorator" ) register_transport_proto("brokenrename+") register_lazy_transport( "brokenrename+", "dromedary.brokenrename", "BrokenRenameTransportDecorator" ) register_transport_proto("vfat+") register_lazy_transport("vfat+", "dromedary.fakevfat", "FakeVFATTransportDecorator") dromedary-0.1.1/_bedding.py000064400000000000000000000023231046102023000137160ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Filesystem path integration points for dromedary. Embedders should replace these functions to control file locations. The defaults use XDG-style paths. """ import os def config_dir(): """Return the configuration directory path.""" xdg = os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")) return os.path.join(xdg, "breezy") def ensure_config_dir_exists(): """Ensure the config directory exists.""" os.makedirs(config_dir(), exist_ok=True) dromedary-0.1.1/_config.py000064400000000000000000000030551046102023000135720ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Configuration integration points for dromedary. Embedders should replace these functions to provide config/auth. The defaults provide basic functionality using the standard library. """ def get_ssh_vendor_name(): """Return the configured SSH vendor name, or None for auto-detect.""" return None def get_auth_user(scheme, host, port=None, default=None, ask=False, prompt=None): """Get username for authentication. Default: returns default, or falls back to the system username. """ if default is not None: return default import getpass return getpass.getuser() def get_auth_password(scheme, host, user, port=None): """Get password for authentication. Default: prompts via getpass. """ import getpass return getpass.getpass(f"Password for {user}@{host}: ") dromedary-0.1.1/_hooks.py000064400000000000000000000073661046102023000134610ustar 00000000000000# Copyright (C) 2007-2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Minimal standalone hooks implementation for dromedary.""" class HookPoint: """A named hook point that maintains a list of callbacks.""" def __init__(self, name, doc, introduced=None, deprecated=None): self.name = name self.__doc__ = doc self.introduced = introduced self.deprecated = deprecated self._callbacks = [] def __iter__(self): return iter(self._callbacks) def __len__(self): return len(self._callbacks) def __repr__(self): return f"" def docs(self): """Generate plain-text documentation for this hook point.""" import textwrap strings = [self.name, "~" * len(self.name), ""] introduced_string = ( ".".join(str(p) for p in self.introduced) if self.introduced else "unknown" ) strings.append(f"Introduced in: {introduced_string}") if self.deprecated: deprecated_string = ".".join(str(p) for p in self.deprecated) strings.append(f"Deprecated in: {deprecated_string}") strings.append("") if self.__doc__: strings.extend(textwrap.wrap(self.__doc__, break_long_words=False)) strings.append("") return "\n".join(strings) class Hooks(dict): """A dict mapping hook names to HookPoint instances.""" def __init__(self): dict.__init__(self) self._callable_names = {} def add_hook(self, name, doc, introduced, deprecated=None): """Register a new hook point.""" self[name] = HookPoint(name, doc, introduced=introduced, deprecated=deprecated) def docs(self): """Generate plain-text documentation for all registered hooks.""" hook_docs = [] cls_name = self.__class__.__name__ hook_docs.append(cls_name) hook_docs.append("-" * len(cls_name)) hook_docs.append("") for hook_name in sorted(self.keys()): hook_docs.append(self[hook_name].docs()) return "\n".join(hook_docs) def install_named_hook(self, hook_name, a_callable, name): """Install a callable on the named hook point.""" try: hook = self[hook_name] except KeyError: raise KeyError(f"Unknown hook: {hook_name!r}") from None hook._callbacks.append(a_callable) if name is not None: self._callable_names[a_callable] = name def uninstall_named_hook(self, hook_name, label): """Remove a callable from the named hook point by label.""" hook = self[hook_name] for i, cb in enumerate(hook._callbacks): if self._callable_names.get(cb) == label: del hook._callbacks[i] del self._callable_names[cb] return raise KeyError(f"No hook named {label!r} on {hook_name!r}") def get_hook_name(self, a_callable): """Return the name associated with a callable, or a repr.""" return self._callable_names.get(a_callable, repr(a_callable)) dromedary-0.1.1/_ui.py000064400000000000000000000030651046102023000127430ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """UI integration points for dromedary. Embedders should replace these functions to integrate with their UI. The defaults provide basic functionality using the standard library. """ def report_transport_activity(transport, byte_count, direction): """Called during transport I/O to report activity. Default: no-op.""" pass def get_password(prompt="", **kwargs): """Prompt for a password. Default: uses getpass.""" import getpass if kwargs: prompt = prompt % kwargs return getpass.getpass(prompt) def get_username(prompt, **kwargs): """Prompt for a username. Default: uses input().""" if kwargs: prompt = prompt % kwargs return input(prompt) def show_message(msg): """Show a message to the user. Default: print to stderr.""" import sys print(msg, file=sys.stderr) dromedary-0.1.1/brokenrename.py000064400000000000000000000022431046102023000146340ustar 00000000000000# Copyright (C) 2007 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport decorator that simulates a transport with broken rename detection.""" from dromedary._transport_rs.brokenrename import BrokenRenameTransportDecorator __all__ = ["BrokenRenameTransportDecorator", "get_test_permutations"] def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(BrokenRenameTransportDecorator, test_server.BrokenRenameServer)] dromedary-0.1.1/cethread.py000064400000000000000000000164161046102023000137520ustar 00000000000000# Copyright (C) 2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Thread implementation that captures and re-raises exceptions. This module provides a thread class that catches exceptions occurring during thread execution and re-raises them when the thread is joined, allowing for better error handling in multi-threaded applications. """ import sys import threading from collections.abc import Callable class CatchingExceptionThread(threading.Thread): """A thread that keeps track of exceptions. If an exception occurs during the thread execution, it's caught and re-raised when the thread is joined(). """ ignored_exceptions: Callable[[Exception], bool] | None def __init__(self, *args, **kwargs): """Initialize a CatchingExceptionThread instance. Args: *args: Positional arguments passed to threading.Thread. **kwargs: Keyword arguments passed to threading.Thread, with special handling for: sync_event: An optional threading.Event used for synchronization. If not provided, a new Event will be created. This event is used to coordinate exception handling between threads. Note: The sync_event is particularly useful when the calling thread must wait for this thread to reach a certain state. If an exception occurs, the event will be set to unblock the waiting thread. """ # There are cases where the calling thread must wait, yet, if an # exception occurs, the event should be set so the caller is not # blocked. The main example is a calling thread that want to wait for # the called thread to be in a given state before continuing. try: sync_event = kwargs.pop("sync_event") except KeyError: # If the caller didn't pass a specific event, create our own sync_event = threading.Event() super().__init__(*args, **kwargs) self.set_sync_event(sync_event) self.exception = None self.ignored_exceptions = None # see set_ignored_exceptions self.lock = threading.Lock() def set_sync_event(self, event): """Set the ``sync_event`` event used to synchronize exception catching. When the thread uses an event to synchronize itself with another thread (setting it when the other thread can wake up from a ``wait`` call), the event must be set after catching an exception or the other thread will hang. Some threads require multiple events and should set the relevant one when appropriate. Note that the event should be initially cleared so the caller can wait() on him and be released when the thread set the event. Also note that the thread can use multiple events, setting them as it progress, while the caller can chose to wait on any of them. What matters is that there is always one event set so that the caller is always released when an exception is caught. Re-using the same event is therefore risky as the thread itself has no idea about which event the caller is waiting on. If the caller has already been released then a cleared event won't guarantee that the caller is still waiting on it. """ self.sync_event = event def switch_and_set(self, new): """Switch to a new ``sync_event`` and set the current one. Using this method protects against race conditions while setting a new ``sync_event``. Note that this allows a caller to wait either on the old or the new event depending on whether it wants a fine control on what is happening inside a thread. :param new: The event that will become ``sync_event`` """ cur = self.sync_event self.lock.acquire() try: # Always release the lock try: self.set_sync_event(new) # From now on, any exception will be synced with the new event except BaseException: # Unlucky, we couldn't set the new sync event, try restoring a # safe state self.set_sync_event(cur) raise # Setting the current ``sync_event`` will release callers waiting # on it, note that it will also be set in run() if an exception is # raised cur.set() finally: self.lock.release() def set_ignored_exceptions( self, ignored: Callable[[Exception], bool] | None | list[type[Exception]] | type[Exception], ): """Declare which exceptions will be ignored. :param ignored: Can be either: - None: all exceptions will be raised, - an exception class: the instances of this class will be ignored, - a tuple of exception classes: the instances of any class of the list will be ignored, - a callable: that will be passed the exception object and should return True if the exception should be ignored """ if ignored is None: self.ignored_exceptions = None elif isinstance(ignored, (Exception, tuple)): self.ignored_exceptions = lambda e: isinstance(e, ignored) elif isinstance(ignored, list): self.ignored_exceptions = lambda e: isinstance(e, tuple(ignored)) # type: ignore else: self.ignored_exceptions = ignored # type: ignore def run(self): """Overrides Thread.run to capture any exception.""" self.sync_event.clear() try: try: super().run() except BaseException: self.exception = sys.exc_info() finally: # Make sure the calling thread is released self.sync_event.set() def join(self, timeout=None): """Overrides Thread.join to raise any exception caught. Calling join(timeout=0) will raise the caught exception or return None if the thread is still alive. """ super().join(timeout) if self.exception is not None: _exc_class, exc_value, _exc_tb = self.exception self.exception = None # The exception should be raised only once if self.ignored_exceptions is None or not self.ignored_exceptions( exc_value ): # Raise non ignored exceptions raise exc_value def pending_exception(self): """Raise the caught exception. This does nothing if no exception occurred. """ self.join(timeout=0) dromedary-0.1.1/chroot.py000064400000000000000000000036031046102023000134630ustar 00000000000000# Copyright (C) 2006-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport that prevents access to locations above a set root. """ from dromedary import pathfilter, register_transport from dromedary._transport_rs.pathfilter import ChrootTransport __all__ = ["ChrootServer", "ChrootTransport", "get_test_permutations"] class ChrootServer(pathfilter.PathFilteringServer): """User space 'chroot' facility. PathFilteringServer does all the path sanitation needed to enforce a chroot, so this is a simple subclass of PathFilteringServer that ignores filter_func. """ def __init__(self, backing_transport): """Initialize the ChrootServer.""" pathfilter.PathFilteringServer.__init__(self, backing_transport, None) def _factory(self, url): return ChrootTransport(self, url) def start_server(self): """Start the chroot server and register its transport.""" self.scheme = "chroot-%d:///" % id(self) register_transport(self.scheme, self._factory) def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(ChrootTransport, test_server.TestingChrootServer)] dromedary-0.1.1/decorator.py000064400000000000000000000161551046102023000141550ustar 00000000000000# Copyright (C) 2006-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport that decorates another transport. This does not change the transport behaviour at all, but provides all the stub functions to allow other decorators to be written easily. """ from dromedary import ( Transport, get_transport_from_path, get_transport_from_url, urlutils, ) class TransportDecorator(Transport): """A no-change decorator for Transports. Subclasses of this are new transports that are based on an underlying transport and can override or intercept some behavior. For example ReadonlyTransportDecorator prevents all write attempts, and FakeNFSTransportDecorator simulates some NFS quirks. This decorator class is not directly usable as a decorator: you must use a subclass which has overridden the _get_url_prefix() class method to return the url prefix for the subclass. """ def __init__(self, url, _decorated=None, _from_transport=None): """Set the 'base' path of the transport. :param _decorated: A private parameter for cloning. :param _from_transport: Is available for subclasses that need to share state across clones. """ prefix = self._get_url_prefix() if not url.startswith(prefix): raise ValueError( f"url {url!r} doesn't start with decorator prefix {prefix!r}" ) not_decorated_url = url[len(prefix) :] if _decorated is None: if urlutils.is_url(not_decorated_url): self._decorated = get_transport_from_url(not_decorated_url) else: self._decorated = get_transport_from_path(not_decorated_url) else: self._decorated = _decorated super().__init__(prefix + self._decorated.base) def abspath(self, relpath): """See Transport.abspath().""" return self._get_url_prefix() + self._decorated.abspath(relpath) def append_file(self, relpath, f, mode=None): """See Transport.append_file().""" return self._decorated.append_file(relpath, f, mode=mode) def append_bytes(self, relpath, bytes, mode=None): """See Transport.append_bytes().""" return self._decorated.append_bytes(relpath, bytes, mode=mode) def _can_roundtrip_unix_modebits(self): """See Transport._can_roundtrip_unix_modebits().""" return self._decorated._can_roundtrip_unix_modebits() def clone(self, offset=None): """See Transport.clone().""" decorated_clone = self._decorated.clone(offset) return self.__class__( self._get_url_prefix() + decorated_clone.base, decorated_clone, self ) def delete(self, relpath): """See Transport.delete().""" return self._decorated.delete(relpath) def delete_tree(self, relpath): """See Transport.delete_tree().""" return self._decorated.delete_tree(relpath) def external_url(self): """See dromedary.Transport.external_url.""" # while decorators are in-process only, they # can be handed back into breezy safely, so # its just the base. return self.base @classmethod def _get_url_prefix(self): """Return the URL prefix of this decorator.""" raise NotImplementedError(self._get_url_prefix) def get(self, relpath): """See Transport.get().""" return self._decorated.get(relpath) def get_smart_client(self): """See Transport.get_smart_client.""" return self._decorated.get_smart_client() def has(self, relpath): """See Transport.has().""" return self._decorated.has(relpath) def is_readonly(self): """See Transport.is_readonly.""" return self._decorated.is_readonly() def mkdir(self, relpath, mode=None): """See Transport.mkdir().""" return self._decorated.mkdir(relpath, mode) def open_write_stream(self, relpath, mode=None): """See Transport.open_write_stream.""" return self._decorated.open_write_stream(relpath, mode=mode) def put_file(self, relpath, f, mode=None): """See Transport.put_file().""" return self._decorated.put_file(relpath, f, mode) def put_bytes(self, relpath, bytes, mode=None): """See Transport.put_bytes().""" return self._decorated.put_bytes(relpath, bytes, mode) def listable(self): """See Transport.listable.""" return self._decorated.listable() def iter_files_recursive(self): """See Transport.iter_files_recursive().""" return self._decorated.iter_files_recursive() def list_dir(self, relpath): """See Transport.list_dir().""" return self._decorated.list_dir(relpath) def _readv(self, relpath, offsets): """See Transport._readv.""" return self._decorated._readv(relpath, offsets) def recommended_page_size(self): """See Transport.recommended_page_size().""" return self._decorated.recommended_page_size() def rename(self, rel_from, rel_to): """See Transport.rename.""" return self._decorated.rename(rel_from, rel_to) def rmdir(self, relpath): """See Transport.rmdir.""" return self._decorated.rmdir(relpath) def _get_segment_parameters(self): return self._decorated.segment_parameters def _set_segment_parameters(self, value): self._decorated.segment_parameters = value segment_parameters = property( _get_segment_parameters, _set_segment_parameters, doc="See Transport.segment_parameters", ) def stat(self, relpath): """See Transport.stat().""" return self._decorated.stat(relpath) def lock_read(self, relpath): """See Transport.lock_read.""" return self._decorated.lock_read(relpath) def lock_write(self, relpath): """See Transport.lock_write.""" return self._decorated.lock_write(relpath) def _redirected_to(self, source, target): redirected = self._decorated._redirected_to(source, target) if redirected is not None: return self.__class__(self._get_url_prefix() + redirected.base, redirected) else: return None def get_test_permutations(): """Return the permutations to be used in testing. The Decorator class is not directly usable, and testing it would not have any benefit - its the concrete classes which need to be tested. """ return [] dromedary-0.1.1/errors.py000064400000000000000000000341051046102023000135020ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Exception classes for dromedary transport layer.""" class TransportError(Exception): """Base class for transport-related errors.""" internal_error = False _fmt = "Transport error: %(msg)s %(orig_error)s" def __init__(self, msg=None, orig_error=None): """Initialize with an optional message and originating error.""" if msg is None and orig_error is not None: msg = str(orig_error) if orig_error is None: orig_error = "" if msg is None: msg = "" self.msg = msg self.orig_error = orig_error Exception.__init__(self) def _get_format_string(self): return self._fmt def __str__(self): """Return the formatted error message.""" fmt = self._get_format_string() if fmt is not None: d = dict(self.__dict__) try: return fmt % d except (KeyError, TypeError): pass if self.args: return str(self.args[0]) return self.msg or "" def __eq__(self, other): """Return True if both errors are of the same class and have equal state.""" if self.__class__ is not other.__class__: return NotImplemented return self.__dict__ == other.__dict__ def __hash__(self): """Return a hash based on object identity.""" return id(self) def __repr__(self): """Return a debug representation including the instance dict.""" return f"<{self.__class__.__name__}({self.__dict__!r})>" class PathError(TransportError): """Generic path-related error.""" _fmt = "Generic path error: %(path)r%(extra)s)" def __init__(self, path, extra=None): """Initialize with the offending path and optional extra detail.""" TransportError.__init__(self) self.path = path if extra: self.extra = ": " + str(extra) else: self.extra = "" class NotADirectory(PathError): """Raised when a path is expected to be a directory but is not.""" _fmt = '"%(path)s" is not a directory %(extra)s' class DirectoryNotEmpty(PathError): """Raised when an operation requires an empty directory.""" _fmt = 'Directory not empty: "%(path)s"%(extra)s' class ResourceBusy(PathError): """Raised when the target resource is currently busy.""" _fmt = 'Device or resource busy: "%(path)s"%(extra)s' class PermissionDenied(PathError): """Raised when access to a path is denied.""" _fmt = 'Permission denied: "%(path)s"%(extra)s' class NoSuchFile(PathError): """Raised when a referenced file or directory does not exist.""" _fmt = 'No such file or directory: "%(path)s"%(extra)s' class FileExists(PathError): """Raised when a file unexpectedly already exists.""" _fmt = 'File exists: "%(path)s"%(extra)s' class UnsupportedProtocol(PathError): """Raised when no transport supports the URL's protocol.""" _fmt = 'Unsupported protocol for url "%(path)s"%(extra)s' class ReadError(PathError): """Raised when reading from a path fails.""" _fmt = "Error reading from %(path)r%(extra)s." class ShortReadvError(PathError): """Raised when a readv call returned fewer bytes than requested.""" _fmt = ( "readv() read %(actual)s bytes rather than %(length)s bytes" ' at %(offset)s for "%(path)s"%(extra)s' ) internal_error = True def __init__(self, path, offset, length, actual, extra=None): """Initialize with the path, requested offset/length and actual bytes read.""" PathError.__init__(self, path, extra=extra) self.offset = offset self.length = length self.actual = actual class PathNotChild(PathError, ValueError): """Raised when a path is not a descendant of an expected base path.""" _fmt = 'Path "%(path)s" is not a child of path "%(base)s"%(extra)s' internal_error = False def __init__(self, path, base, extra=None): """Initialize with the path, expected base path and optional extra detail.""" TransportError.__init__(self) self.path = path self.base = base if extra: self.extra = ": " + str(extra) else: self.extra = "" class TransportNotPossible(TransportError): """Raised when an operation is not supported by the transport.""" _fmt = "Transport operation not possible: %(msg)s %(orig_error)s" class NotLocalUrl(TransportError): """Raised when a URL was expected to refer to a local path but does not.""" _fmt = "%(url)s is not a local path." def __init__(self, url): """Initialize with the offending URL.""" self.url = url TransportError.__init__(self) class NoSmartMedium(TransportError): """Raised when a transport cannot tunnel the smart protocol.""" _fmt = "The transport '%(transport)s' cannot tunnel the smart protocol." internal_error = True def __init__(self, transport): """Initialize with the transport that lacks smart-protocol support.""" self.transport = transport TransportError.__init__(self) class DependencyNotPresent(TransportError): """A required dependency for a transport is not present.""" _fmt = 'Unable to import library "%(library)s": %(error)s' def __init__(self, library, error): """Initialize with the missing library name and import error.""" self.library = library self.error = error TransportError.__init__(self) class RedirectRequested(TransportError): """Raised when the server requested a redirect to another URL.""" _fmt = "%(source)s is%(permanently)s redirected to %(target)s" def __init__(self, source, target, is_permanent=False): """Initialize with the source URL, target URL and whether permanent.""" self.source = source self.target = target if is_permanent: self.permanently = " permanently" else: self.permanently = "" TransportError.__init__(self) class TooManyRedirections(TransportError): """Raised when the maximum redirect chain length was exceeded.""" _fmt = "Too many redirections" class InProcessTransport(TransportError): """Raised when a transport can only be reached from within this process.""" _fmt = "The transport '%(transport)s' is only accessible within this process." def __init__(self, transport): """Initialize with the in-process-only transport.""" self.transport = transport TransportError.__init__(self) class ConnectionError(TransportError): """Raised when a transport connection fails.""" _fmt = "Connection error: %(msg)s" class UnusableRedirect(TransportError): """Raised when a redirect cannot be followed.""" _fmt = "Unable to follow redirect from %(source)s to %(target)s: %(reason)s." def __init__(self, source, target, reason): """Initialize with the source URL, target URL and reason.""" TransportError.__init__(self) self.source = source self.target = target self.reason = reason # HTTP-specific errors class InvalidHttpResponse(TransportError): """Raised when an HTTP response could not be parsed or was unexpected.""" _fmt = "Invalid http response for %(path)s: %(msg)s%(orig_error)s" def __init__(self, path, msg, orig_error=None, headers=None): """Initialize with the path, message, original error and headers.""" self.path = path if orig_error is None: orig_error = "" else: orig_error = f": {orig_error!r}" self.headers = headers TransportError.__init__(self, msg, orig_error=orig_error) class UnexpectedHttpStatus(InvalidHttpResponse): """Raised when an HTTP response had an unexpected status code.""" _fmt = "Unexpected HTTP status %(code)d for %(path)s: %(extra)s" def __init__(self, path, code, extra=None, headers=None): """Initialize with the path, HTTP status code, optional extra and headers.""" self.path = path self.code = code self.extra = extra or "" full_msg = "status code %d unexpected" % code if extra is not None: full_msg += ": " + extra InvalidHttpResponse.__init__(self, path, full_msg, headers=headers) class InvalidHttpRange(InvalidHttpResponse): """Raised when an HTTP range request returned an invalid range.""" _fmt = "Invalid http range %(range)r for %(path)s: %(msg)s" def __init__(self, path, range, msg): """Initialize with the path, requested range and message.""" self.range = range InvalidHttpResponse.__init__(self, path, msg) class HttpBoundaryMissing(InvalidHttpResponse): """Raised when a multipart HTTP response is missing its MIME boundary.""" _fmt = "HTTP MIME Boundary missing for %(path)s: %(msg)s" def __init__(self, path, msg): """Initialize with the path and message.""" InvalidHttpResponse.__init__(self, path, msg) class BadHttpRequest(UnexpectedHttpStatus): """Raised when the server reported a bad HTTP request.""" _fmt = "Bad http request for %(path)s: %(reason)s" def __init__(self, path, reason): """Initialize with the path and reason.""" self.path = path self.reason = reason TransportError.__init__(self, reason) class InvalidRange(TransportError): """Raised when a range read targets an invalid offset.""" _fmt = "Invalid range access in %(path)s at %(offset)s: %(msg)s" def __init__(self, path, offset, msg=None): """Initialize with the path, offset and optional message.""" TransportError.__init__(self, msg) self.path = path self.offset = offset # Smart protocol errors class SmartProtocolError(TransportError): """Generic error in the bzr smart protocol.""" _fmt = "Generic bzr smart protocol error: %(details)s" def __init__(self, details): """Initialize with the protocol error details.""" self.details = details TransportError.__init__(self) class ErrorFromSmartServer(TransportError): """An error tuple was received from a smart server.""" _fmt = "Error received from smart server: %(error_tuple)r" internal_error = True def __init__(self, error_tuple): """Initialize with the raw error tuple from the smart server.""" self.error_tuple = error_tuple try: self.error_verb = error_tuple[0] except IndexError: self.error_verb = None self.error_args = error_tuple[1:] TransportError.__init__(self) class UnexpectedSmartServerResponse(TransportError): """The smart server returned a response that could not be understood.""" _fmt = "Could not understand response from smart server: %(response_tuple)r" def __init__(self, response_tuple): """Initialize with the unexpected response tuple.""" self.response_tuple = response_tuple TransportError.__init__(self) class UnknownSmartMethod(TransportError): """The smart server did not recognise the requested verb.""" _fmt = "The server does not recognise the '%(verb)s' request." internal_error = True def __init__(self, verb): """Initialize with the unrecognised verb.""" self.verb = verb TransportError.__init__(self) # File-level locking errors raised by transport implementations. # # These class names are imported by the Rust extensions (see # dromedary/_transport_rs/src/lib.rs), so they must stay at module level. # Higher-level lock concepts (repository/branch/working-tree locks) belong # in the consuming application (e.g. breezy.errors), which translates these # at the boundary if it wants to surface them as its own lock errors. class LockContention(TransportError): """Raised when a lock is held by another process.""" _fmt = 'Could not acquire lock "%(lock)s": %(msg)s' internal_error = False def __init__(self, lock, msg=""): """Initialize with the contended lock and optional message.""" self.lock = lock self.msg = msg TransportError.__init__(self) class LockFailed(TransportError): """Raised when acquiring a lock fails for reasons other than contention.""" internal_error = False _fmt = "Cannot lock %(lock)s: %(why)s" def __init__(self, lock, why): """Initialize with the lock and the reason it could not be acquired.""" self.lock = lock self.why = why TransportError.__init__(self) class SocketConnectionError(ConnectionError): """Socket connection error.""" _fmt = "%(formatted_msg)s" def __init__(self, host, port=None, msg=None, orig_error=None): """Initialize with the host, optional port, message and originating error.""" if msg is None: msg = "Failed to connect to" orig_error = "" if orig_error is None else "; " + str(orig_error) self.host = host port = "" if port is None else f":{port}" self.port = port self.formatted_msg = f"{msg} {host}{port}{orig_error}" ConnectionError.__init__(self, self.formatted_msg) class StrangeHostname(TransportError): """Refusing to connect to strange SSH hostname.""" _fmt = "Refusing to connect to strange SSH hostname %(hostname)s" def __init__(self, hostname): """Initialize with the rejected hostname.""" self.hostname = hostname TransportError.__init__(self) dromedary-0.1.1/fakenfs.py000064400000000000000000000021641046102023000136030ustar 00000000000000# Copyright (C) 2005, 2006, 2008 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport decorator simulating NFS quirks.""" from dromedary._transport_rs.fakenfs import FakeNFSTransportDecorator __all__ = ["FakeNFSTransportDecorator", "get_test_permutations"] def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(FakeNFSTransportDecorator, test_server.FakeNFSServer)] dromedary-0.1.1/fakevfat.py000064400000000000000000000021771046102023000137610ustar 00000000000000# Copyright (C) 2006 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport decorator simulating VFAT filesystem restrictions.""" from dromedary._transport_rs.fakevfat import FakeVFATTransportDecorator __all__ = ["FakeVFATTransportDecorator", "get_test_permutations"] def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(FakeVFATTransportDecorator, test_server.FakeVFATServer)] dromedary-0.1.1/gio_transport.py000064400000000000000000000042421046102023000150570ustar 00000000000000# Copyright (C) 2010 Canonical Ltd. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # # Author: Mattias Eriksson """Implementation of Transport over gio. It provides the gio+XXX:// protocols where XXX is any of the protocols supported by gio (file, sftp, smb, dav, ftp, ssh, obex). The transport is implemented in Rust against the gtk-rs `gio` crate, gated behind a non-default `gio` Cargo feature. When dromedary is built without that feature, importing this module raises DependencyNotPresent to match the historical behaviour when the legacy Python `gio` module was missing. """ from dromedary import urlutils from dromedary.errors import DependencyNotPresent from dromedary.tests.test_server import TestServer try: from dromedary._transport_rs.gio import GioTransport except ImportError as e: raise DependencyNotPresent("gio", e) from e __all__ = ["GioLocalURLServer", "GioTransport", "get_test_permutations"] class GioLocalURLServer(TestServer): """A pretend server for local transports, using gio+file:// urls. Of course no actual server is required to access the local filesystem, so this just exists to tell the test code how to get to it. """ def start_server(self): """Start the server (no-op for local filesystem access).""" pass def get_url(self): """See Transport.Server.get_url.""" return "gio+" + urlutils.local_path_to_url("") def get_test_permutations(): """Return the permutations to be used in testing.""" return [(GioTransport, GioLocalURLServer)] dromedary-0.1.1/http/__init__.py000064400000000000000000000112071046102023000147020ustar 00000000000000# Copyright (C) 2005-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Base implementation of Transport over http. There are separate implementation modules for each http client implementation. """ DEBUG = 0 import os import sys from dromedary.version import version_string as dromedary_version _user_agent_prefix = f"Dromedary/{dromedary_version}" def set_user_agent(prefix): """Set the User-Agent prefix for HTTP requests. Args: prefix: The User-Agent string to use, e.g. "Breezy/3.4.0". """ global _user_agent_prefix _user_agent_prefix = prefix def _default_credential_lookup(protocol, host, port=None, path=None, realm=None): """Default credential lookup returning no credentials. Override via set_credential_lookup() to integrate with a credential store. Returns: tuple: (user, password) or (None, None) if no credentials found. """ return None, None _credential_lookup = _default_credential_lookup def set_credential_lookup(func): """Set the function used to look up HTTP credentials. Args: func: A callable(protocol, host, port=None, path=None, realm=None) returning (user, password) or (None, None). """ global _credential_lookup _credential_lookup = func def get_credentials(protocol, host, port=None, path=None, realm=None): """Look up stored credentials for an HTTP connection.""" return _credential_lookup(protocol, host, port=port, path=path, realm=realm) def default_user_agent(): """Get the default User-Agent string for HTTP requests.""" return _user_agent_prefix # Note for packagers: if there is no package providing certs for your platform, # the curl project produces http://curl.haxx.se/ca/cacert.pem weekly. _ssl_ca_certs_known_locations = [ "/etc/ssl/certs/ca-certificates.crt", # Ubuntu/debian/gentoo "/etc/pki/tls/certs/ca-bundle.crt", # Fedora/CentOS/RH "/etc/ssl/ca-bundle.pem", # OpenSuse "/etc/ssl/cert.pem", # OpenSuse "/usr/local/share/certs/ca-root-nss.crt", # FreeBSD # XXX: Needs checking, can't trust the interweb ;) -- vila 2012-01-25 "/etc/openssl/certs/ca-certificates.crt", # Solaris ] def default_ca_certs(): """Get the default path to CA certificates for SSL verification. Searches for CA certificate bundles in platform-specific locations. On Windows, looks for cacert.pem in the executable's directory. On other platforms, searches a list of known locations and returns the first existing path. Returns: str: Path to the CA certificate bundle. If no bundle is found, returns the first known location as a default. """ if sys.platform == "win32": return os.path.join(os.path.dirname(sys.executable), "cacert.pem") elif sys.platform == "darwin": # FIXME: Needs some default value for osx, waiting for osx installers # guys feedback -- vila 2012-01-25 pass else: # Try known locations for friendly OSes providing the root certificates # without making them hard to use for any https client. for path in _ssl_ca_certs_known_locations: if os.path.exists(path): # First found wins return path # A default path that makes sense and will be mentioned in the error # presented to the user, even if not correct for all platforms return _ssl_ca_certs_known_locations[0] def default_cert_reqs(): """Get the default certificate verification requirement for the platform. On Windows and macOS, returns ssl.CERT_NONE due to lack of native access to root certificates. On other platforms, returns ssl.CERT_REQUIRED. """ import ssl if sys.platform in ("win32", "darwin"): # FIXME: Once we get a native access to root certificates there, this # won't needed anymore. See http://pad.lv/920455 -- vila 2012-02-15 return ssl.CERT_NONE else: return ssl.CERT_REQUIRED ssl_ca_certs = default_ca_certs ssl_cert_reqs = default_cert_reqs dromedary-0.1.1/http/ca_bundle.py000064400000000000000000000052611046102023000150620ustar 00000000000000# Copyright (C) 2007 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Auto-detect of CA bundle for SSL connections.""" import logging import os import sys logger = logging.getLogger("dromedary.http.ca_bundle") _ca_path = None def get_ca_path(use_cache=True): """Return location of CA bundle.""" global _ca_path if _ca_path is not None and use_cache: return _ca_path # Find CA bundle for SSL # Reimplementation in Python the magic of curl command line tool # from "Details on Server SSL Certificates" # http://curl.haxx.se/docs/sslcerts.html # # 4. If you're using the curl command line tool, you can specify your own # CA cert path by setting the environment variable CURL_CA_BUNDLE to the # path of your choice. # # If you're using the curl command line tool on Windows, curl will # search for a CA cert file named "curl-ca-bundle.crt" in these # directories and in this order: # 1. application's directory # 2. current working directory # 3. Windows System directory (e.g. C:\windows\system32) # 4. Windows Directory (e.g. C:\windows) # 5. all directories along %PATH% # # NOTES: # bialix: Windows directories usually listed in PATH env variable # j-a-meinel: bzr should not look in current working dir path = os.environ.get("CURL_CA_BUNDLE") if not path and sys.platform == "win32": dirs = [os.path.realpath(os.path.dirname(sys.argv[0]))] # app dir paths = os.environ.get("PATH") if paths: # don't include the cwd in the search paths = [i for i in paths.split(os.pathsep) if i not in ("", ".")] dirs.extend(paths) for d in dirs: fname = os.path.join(d, "curl-ca-bundle.crt") if os.path.isfile(fname): path = fname break if path: logger.debug("using CA bundle: %r", path) else: path = "" if use_cache: _ca_path = path return path dromedary-0.1.1/http/response.py000064400000000000000000000455631046102023000150150ustar 00000000000000# Copyright (C) 2006-2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Handlers for HTTP Responses. The purpose of these classes is to provide a uniform interface for clients to standard HTTP responses, single range responses and multipart range responses. """ import email.utils as email_utils import http.client as http_client import os from io import BytesIO from dromedary.errors import ( HttpBoundaryMissing, InvalidHttpRange, InvalidHttpResponse, InvalidRange, ShortReadvError, UnexpectedHttpStatus, ) from dromedary.osutils import pumpfile class ResponseFile: """A wrapper around the http socket containing the result of a GET request. Only read() and seek() (forward) are supported. """ def __init__(self, path, infile): """Constructor. :param path: File url, for error reports. :param infile: File-like socket set at body start. """ self._path = path self._file = infile self._pos = 0 def close(self): """Close this file. Dummy implementation for consistency with the 'file' API. """ def __enter__(self): """Enter the runtime context for the ResponseFile. Returns: ResponseFile: This object for use in with statements. """ return self def __exit__(self, exc_type, exc_val, exc_tb): """Exit the runtime context for the ResponseFile. Args: exc_type: The exception type if an exception occurred. exc_val: The exception value if an exception occurred. exc_tb: The exception traceback if an exception occurred. Returns: bool: False to propagate any exceptions that occurred. """ return False # propogate exceptions. def read(self, size=None): """Read size bytes from the current position in the file. :param size: The number of bytes to read. Leave unspecified or pass -1 to read to EOF. """ data = self._file.read(size) self._pos += len(data) return data def readline(self): """Read a single line from the current position in the file. Returns: bytes: A line of data including the newline character, or an empty bytes object if at EOF. """ data = self._file.readline() self._pos += len(data) return data def readlines(self, size=None): """Read all remaining lines from the current position. Args: size: Optional hint for the number of bytes to read. If provided, reading may stop after approximately this many bytes. Returns: list[bytes]: A list of lines including newline characters. """ data = self._file.readlines() self._pos += sum(map(len, data)) return data def __iter__(self): """Iterate over the lines in the file. Yields: bytes: Each line in the file including newline characters. """ while True: line = self.readline() if not line: return yield line def tell(self): """Return the current position in the file. Returns: int: The current byte position in the file. """ return self._pos def seek(self, offset, whence=os.SEEK_SET): """Seek to a new position in the file. Only forward seeking is supported. Attempting to seek backwards will raise an AssertionError. Args: offset: The number of bytes to seek. whence: How to interpret the offset. Only SEEK_SET (absolute) and SEEK_CUR (relative to current position) are supported. Raises: AssertionError: If attempting to seek backwards or using an unsupported whence value. """ if whence == os.SEEK_SET: if offset < self._pos: raise AssertionError( f"Can't seek backwards, pos: {self._pos}, offset: {offset}" ) to_discard = offset - self._pos elif whence == os.SEEK_CUR: to_discard = offset else: raise AssertionError("Can't seek backwards") if to_discard: # Just discard the unwanted bytes self.read(to_discard) # A RangeFile expects the following grammar (simplified to outline the # assumptions we rely upon). # file: single_range # | multiple_range # single_range: content_range_header data # multiple_range: boundary_header boundary (content_range_header data boundary)+ class RangeFile(ResponseFile): """File-like object that allow access to partial available data. All accesses should happen sequentially since the acquisition occurs during an http response reception (as sockets can't be seeked, we simulate the seek by just reading and discarding the data). The access pattern is defined by a set of ranges discovered as reading progress. Only one range is available at a given time, so all accesses should happen with monotonically increasing offsets. """ # in _checked_read() below, we may have to discard several MB in the worst # case. To avoid buffering that much, we read and discard by chunks # instead. The underlying file is either a socket or a BytesIO, so reading # 8k chunks should be fine. _discarded_buf_size = 8192 def __init__(self, path, infile): """Constructor. :param path: File url, for error reports. :param infile: File-like socket set at body start. """ super().__init__(path, infile) self._boundary = None # When using multi parts response, this will be set with the headers # associated with the range currently read. self._headers = None # Default to the whole file of unspecified size self.set_range(0, -1) def set_range(self, start, size): """Change the range mapping. Updates the current range being read from the file. Args: start: The starting byte position of the range. size: The size of the range in bytes. Use -1 for unknown size. """ self._start = start self._size = size # Set the new _pos since that's what we want to expose self._pos = self._start def set_boundary(self, boundary): """Define the boundary used in a multi parts message. The file should be at the beginning of the body, the first range definition is read and taken into account. """ if not isinstance(boundary, bytes): raise TypeError(boundary) self._boundary = boundary # Decode the headers and setup the first range self.read_boundary() self.read_range_definition() def read_boundary(self): """Read the boundary headers defining a new range. Reads and validates the boundary line from a multipart response. Handles additional CRLFs that may precede the boundary as per RFC2616. Raises: HttpBoundaryMissing: If the expected boundary is not found (possibly due to a timeout). InvalidHttpResponse: If the boundary line doesn't match the expected format. """ boundary_line = b"\r\n" while boundary_line == b"\r\n": # RFC2616 19.2 Additional CRLFs may precede the first boundary # string entity. # To be on the safe side we allow it before any boundary line boundary_line = self._file.readline() if boundary_line == b"": # A timeout in the proxy server caused the response to end early. # See launchpad bug 198646. raise HttpBoundaryMissing(self._path, self._boundary) if boundary_line != b"--" + self._boundary + b"\r\n": # email_utils.unquote() incorrectly unquotes strings enclosed in <> # IIS 6 and 7 incorrectly wrap boundary strings in <> # together they make a beautiful bug, which we will be gracious # about here if ( self._unquote_boundary(boundary_line) != b"--" + self._boundary + b"\r\n" ): raise InvalidHttpResponse( self._path, f"Expected a boundary ({self._boundary}) line, got '{boundary_line}'", ) def _unquote_boundary(self, b): """Unquote a boundary string that may be incorrectly quoted. Works around a bug where IIS 6 and 7 incorrectly wrap boundary strings in angle brackets (<>), which email_utils.unquote() handles incorrectly. Args: b: The boundary line as bytes. Returns: bytes: The unquoted boundary line. """ return ( b[:2] + email_utils.unquote(b[2:-2].decode("ascii")).encode("ascii") + b[-2:] ) def read_range_definition(self): """Read a new range definition in a multi parts message. Parse the headers including the empty line following them so that we are ready to read the data itself. """ self._headers = http_client.parse_headers(self._file) # Extract the range definition content_range = self._headers.get("content-range", None) if content_range is None: raise InvalidHttpResponse( self._path, "Content-Range header missing in a multi-part response", headers=self._headers, ) self.set_range_from_header(content_range) def set_range_from_header(self, content_range): """Helper to set the new range from its description in the headers. Parses a Content-Range header and updates the current range accordingly. Args: content_range: The Content-Range header value (e.g., "bytes 200-1023/1024"). Raises: InvalidHttpRange: If the header is malformed, contains an unsupported range type, has invalid values, or specifies a range with size <= 0. """ try: rtype, values = content_range.split() except ValueError as e: raise InvalidHttpRange(self._path, content_range, "Malformed header") from e if rtype != "bytes": raise InvalidHttpRange( self._path, content_range, f"Unsupported range type '{rtype}'" ) try: # We don't need total, but note that it may be either the file size # or '*' if the server can't or doesn't want to return the file # size. start_end, _total = values.split("/") start, end = start_end.split("-") start = int(start) end = int(end) except ValueError as e: raise InvalidHttpRange( self._path, content_range, "Invalid range values" ) from e size = end - start + 1 if size <= 0: raise InvalidHttpRange( self._path, content_range, "Invalid range, size <= 0" ) self.set_range(start, size) def _checked_read(self, size): """Read the file checking for short reads. The data read is discarded along the way. Used internally for seeking forward by discarding unwanted bytes. Args: size: The number of bytes to read and discard. Raises: ShortReadvError: If unable to read the requested number of bytes. """ pos = self._pos remaining = size while remaining > 0: data = self._file.read(min(remaining, self._discarded_buf_size)) remaining -= len(data) if not data: raise ShortReadvError(self._path, pos, size, size - remaining) self._pos += size def _seek_to_next_range(self): """Seek to the next range in a multipart response. Reads the boundary and range definition headers for the next part. Raises: InvalidRange: If no boundary is set (not a multipart response). """ # We will cross range boundaries if self._boundary is None: # If we don't have a boundary, we can't find another range raise InvalidRange( self._path, self._pos, f"Range ({self._start}, {self._size}) exhausted" ) self.read_boundary() self.read_range_definition() def read(self, size=-1): """Read size bytes from the current position in the file. Reading across ranges is not supported. We rely on the underlying http client to clean the socket if we leave bytes unread. This may occur for the final boundary line of a multipart response or for any range request not entirely consumed by the client (due to offset coalescing) :param size: The number of bytes to read. Leave unspecified or pass -1 to read to EOF. """ if self._size > 0 and self._pos == self._start + self._size: if size == 0: return b"" else: self._seek_to_next_range() elif self._pos < self._start: raise InvalidRange( self._path, self._pos, f"Can't read {size} bytes before range ({self._start}, {self._size})", ) if self._size > 0 and size > 0 and self._pos + size > self._start + self._size: raise InvalidRange( self._path, self._pos, f"Can't read {size} bytes across range ({self._start}, {self._size})", ) # read data from file buf = BytesIO() limited = size if self._size > 0: # Don't read past the range definition limited = self._start + self._size - self._pos if size >= 0: limited = min(limited, size) pumpfile(self._file, buf, None if limited < 0 else limited) data = buf.getvalue() # Update _pos respecting the data effectively read self._pos += len(data) return data def seek(self, offset, whence=0): """Seek to a new position in the file. Supports seeking within and across range boundaries in multipart responses. Only forward seeking is supported. Args: offset: The number of bytes to seek. whence: How to interpret the offset: 0 (SEEK_SET): Absolute position 1 (SEEK_CUR): Relative to current position 2 (SEEK_END): Relative to end (only if size is known) Raises: InvalidRange: If attempting to seek backwards, beyond known boundaries, or from end when size is unknown. ValueError: If whence has an invalid value. """ start_pos = self._pos if whence == 0: final_pos = offset elif whence == 1: final_pos = start_pos + offset elif whence == 2: if self._size > 0: final_pos = self._start + self._size + offset # offset < 0 else: raise InvalidRange( self._path, self._pos, "RangeFile: can't seek from end while size is unknown", ) else: raise ValueError(f"Invalid value {whence} for whence.") if final_pos < self._pos: # Can't seek backwards raise InvalidRange( self._path, self._pos, f"RangeFile: trying to seek backwards to {final_pos}", ) if self._size > 0: cur_limit = self._start + self._size while final_pos > cur_limit: # We will cross range boundaries remain = cur_limit - self._pos if remain > 0: # Finish reading the current range self._checked_read(remain) self._seek_to_next_range() cur_limit = self._start + self._size size = final_pos - self._pos if size > 0: # size can be < 0 if we crossed a range boundary # We don't need the data, just read it and throw it away self._checked_read(size) def tell(self): """Return the current position in the file. Returns: int: The current byte position in the file. """ return self._pos def handle_response(url, code, getheader, data): """Interpret the code & headers and wrap the provided data in a RangeFile. This is a factory method which returns an appropriate RangeFile based on the code & headers it's given. :param url: The url being processed. Mostly for error reporting :param code: The integer HTTP response code :param getheader: Function for retrieving header :param data: A file-like object that can be read() to get the requested data :return: A file-like object that can seek()+read() the ranges indicated by the headers. """ if code == 200: # A whole file rfile = ResponseFile(url, data) elif code == 206: rfile = RangeFile(url, data) # When there is no content-type header we treat the response as # being of type 'application/octet-stream' as per RFC2616 section # 7.2.1. # Therefore it is obviously not multipart content_type = getheader("content-type", "application/octet-stream") from email.message import EmailMessage msg = EmailMessage() msg["content-type"] = content_type params = msg["content-type"].params mimetype = msg.get_content_type() if mimetype == "multipart/byteranges": rfile.set_boundary(params["boundary"].encode("ascii")) else: # A response to a range request, but not multipart content_range = getheader("content-range", None) if content_range is None: raise InvalidHttpResponse( url, "Missing the Content-Range header in a 206 range response" ) rfile.set_range_from_header(content_range) else: raise UnexpectedHttpStatus(url, code) return rfile dromedary-0.1.1/http/urllib.py000064400000000000000000003417731046102023000144520ustar 00000000000000# Copyright (C) 2005-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Base implementation of Transport over http using urllib. There are separate implementation modules for each http client implementation. """ DEBUG = 0 import base64 import errno import hashlib import http.client import logging import os import re import socket import ssl import sys import time import urllib import urllib.request from urllib.parse import urlencode, urljoin, urlparse import dromedary as _mod_dromedary from dromedary import ConnectedTransport, urlutils from dromedary.errors import ( BadHttpRequest, ConnectionError, HttpBoundaryMissing, InvalidHttpRange, InvalidHttpResponse, InvalidRange, NoSuchFile, RedirectRequested, ShortReadvError, TransportError, TransportNotPossible, UnexpectedHttpStatus, UnusableRedirect, ) from dromedary.http import default_user_agent from dromedary.osutils import rand_chars logger = logging.getLogger("dromedary.http.urllib") debug_logger = logging.getLogger("dromedary.http") evil_logger = logging.getLogger("dromedary.evil") # TODO: handle_response should be integrated into the http/__init__.py from .response import handle_response # FIXME: Oversimplifying, two kind of exceptions should be # raised, once a request is issued: URLError before we have been # able to process the response, HTTPError after that. Process the # response means we are able to leave the socket clean, so if we # are not able to do that, we should close the connection. The # actual code more or less do that, tests should be written to # ensure that. checked_kerberos = False kerberos = None def splitport(host): """Split a network host and port from a host string. Parses a host string in the format 'hostname:port' and returns the hostname and port as separate values. Args: host: A string containing hostname and optional port number in the format 'hostname:port' or just 'hostname'. Returns: A tuple of (hostname, port) where: - hostname (str): The hostname portion - port (str or None): The port number as a string, or None if no port """ m = re.fullmatch("(.*):([0-9]*)", host, re.DOTALL) if m: host, port = m.groups() return host, port or None return host, None class _ReportingFileSocket: """A wrapper around file-like socket objects that reports activity. This class wraps a file-like socket object and reports read activity through a callback function. It delegates all other operations to the underlying socket object. """ def __init__(self, filesock, report_activity=None): """Initialize a reporting file socket wrapper. Args: filesock: The underlying file-like socket object to wrap. report_activity: Optional callback function to report activity. Should accept (size, direction) parameters. """ self.filesock = filesock self._report_activity = report_activity def report_activity(self, size, direction): """Report activity to the callback function if available. Args: size: The number of bytes involved in the activity. direction: The direction of activity ('read' or 'write'). """ if self._report_activity: self._report_activity(size, direction) def read(self, size=1): """Read up to size bytes from the file socket. Args: size: Maximum number of bytes to read. Returns: Bytes read from the socket. """ s = self.filesock.read(size) self.report_activity(len(s), "read") return s def readline(self, size=-1): """Read a line from the file socket. Args: size: Maximum number of bytes to read, or -1 for unlimited. Returns: A line read from the socket. """ s = self.filesock.readline(size) self.report_activity(len(s), "read") return s def readinto(self, b): """Read data from the file socket into a pre-allocated buffer. Args: b: A pre-allocated buffer to read data into. Returns: Number of bytes read. """ s = self.filesock.readinto(b) self.report_activity(s, "read") return s def __getattr__(self, name): """Delegate unknown attributes to the underlying file socket. Args: name: The attribute name to look up. Returns: The attribute from the underlying file socket. """ return getattr(self.filesock, name) class _ReportingSocket: """A wrapper around socket objects that reports network activity. This class wraps a socket object and reports both read and write activity through a callback function. It provides buffered file creation and delegates all other operations to the underlying socket. """ def __init__(self, sock, report_activity=None): """Initialize a reporting socket wrapper. Args: sock: The underlying socket object to wrap. report_activity: Optional callback function to report activity. Should accept (size, direction) parameters. """ self.sock = sock self._report_activity = report_activity def report_activity(self, size, direction): """Report activity to the callback function if available. Args: size: The number of bytes involved in the activity. direction: The direction of activity ('read' or 'write'). """ if self._report_activity: self._report_activity(size, direction) def sendall(self, s, *args): """Send all data to the socket. Args: s: Data to send. *args: Additional arguments passed to the underlying socket. """ self.sock.sendall(s, *args) self.report_activity(len(s), "write") def recv(self, *args): """Receive data from the socket. Args: *args: Arguments passed to the underlying socket's recv method. Returns: Data received from the socket. """ s = self.sock.recv(*args) self.report_activity(len(s), "read") return s def makefile(self, mode="r", bufsize=-1): """Create a buffered file-like object from the socket. Creates a file-like object with proper buffering to improve performance of readline() operations, which would otherwise read one byte at a time. Args: mode: File mode ('r', 'w', 'b', etc.). bufsize: Buffer size (ignored, uses optimized 64KB buffer). Returns: A _ReportingFileSocket object that wraps the buffered file socket. """ # http.client creates a fileobject that doesn't do buffering, which # makes fp.readline() very expensive because it only reads one byte # at a time. So we wrap the socket in an object that forces # sock.makefile to make a buffered file. fsock = self.sock.makefile(mode, 65536) # And wrap that into a reporting kind of fileobject return _ReportingFileSocket(fsock, self._report_activity) def __getattr__(self, name): """Delegate unknown attributes to the underlying socket. Args: name: The attribute name to look up. Returns: The attribute from the underlying socket. """ return getattr(self.sock, name) # We define our own Response class to keep our http.client pipe clean class Response(http.client.HTTPResponse): """Custom HTTPResponse, to avoid the need to decorate. http.client prefers to decorate the returned objects, rather than using a custom object. """ # Some responses have bodies in which we have no interest _body_ignored_responses = [301, 302, 303, 307, 308, 404, 501] # in finish() below, we may have to discard several MB in the worst # case. To avoid buffering that much, we read and discard by chunks # instead. The underlying file is either a socket or a StringIO, so reading # 8k chunks should be fine. _discarded_buf_size = 8192 def __init__(self, sock, debuglevel=0, method=None, url=None): """Initialize a custom HTTP response object. Args: sock: The socket object for the HTTP connection. debuglevel: Debug level for HTTP debugging output (default: 0). method: HTTP method used for the request (optional). url: URL of the request (optional). """ self.url = url super().__init__(sock, debuglevel=debuglevel, method=method, url=url) def begin(self): """Begin to read the response from the server. http.client assumes that some responses get no content and do not even attempt to read the body in that case, leaving the body in the socket, blocking the next request. Let's try to workaround that. """ http.client.HTTPResponse.begin(self) if self.status in self._body_ignored_responses: if self.debuglevel >= 2: print( "For status: [{}], will ready body, length: {}".format( self.status, self.length ) ) if not (self.length is None or self.will_close): # In some cases, we just can't read the body not # even try or we may encounter a 104, 'Connection # reset by peer' error if there is indeed no body # and the server closed the connection just after # having issued the response headers (even if the # headers indicate a Content-Type...) body = self.read(self.length) if self.debuglevel >= 9: # This one can be huge and is generally not interesting print(f"Consumed body: [{body}]") self.close() elif self.status == 200: # Whatever the request is, it went ok, so we surely don't want to # close the connection. Some cases are not correctly detected by # http.client.HTTPConnection.getresponse (called by # http.client.HTTPResponse.begin). The CONNECT response for the https # through proxy case is one. Note: the 'will_close' below refers # to the "true" socket between us and the server, whereas the # 'close()' above refers to the copy of that socket created by # http.client for the response itself. So, in the if above we close the # socket to indicate that we are done with the response whereas # below we keep the socket with the server opened. self.will_close = False def finish(self): """Finish reading the body. In some cases, the client may have left some bytes to read in the body. That will block the next request to succeed if we use a persistent connection. If we don't use a persistent connection, well, nothing will block the next request since a new connection will be issued anyway. :return: the number of bytes left on the socket (may be None) """ pending = None if not self.isclosed(): # Make sure nothing was left to be read on the socket pending = 0 data = True while data and self.length: # read() will update self.length data = self.read(min(self.length, self._discarded_buf_size)) pending += len(data) if pending: logger.debug("%s bytes left on the HTTP socket", pending) self.close() return pending # Not inheriting from 'object' because http.client.HTTPConnection doesn't. class AbstractHTTPConnection: """A custom HTTP(S) Connection, which can reset itself on a bad response.""" response_class = Response # When we detect a server responding with the whole file to range requests, # we want to warn. But not below a given thresold. _range_warning_thresold = 1024 * 1024 def __init__(self, report_activity=None): """Initialize the abstract HTTP connection. Args: report_activity: Optional callback function to report network activity. Should accept (size, direction) parameters. """ self._response = None self._report_activity = report_activity self._ranges_received_whole_file = None def _mutter_connect(self): """Log connection information for debugging purposes. Outputs connection details including host, port, and proxy information to the trace system for debugging HTTP connections. """ netloc = f"{self.host}:{self.port}" if self.proxied_host is not None: netloc += f"(proxy for {self.proxied_host})" logger.debug("* About to connect() to %s", netloc) def getresponse(self): """Capture the response to be able to cleanup.""" self._response = http.client.HTTPConnection.getresponse(self) return self._response def cleanup_pipe(self): """Read the remaining bytes of the last response if any.""" if self._response is not None: try: pending = self._response.finish() # Warn the user (once) if ( self._ranges_received_whole_file is None and self._response.status == 200 and pending and pending > self._range_warning_thresold ): self._ranges_received_whole_file = True logger.warning( "Got a 200 response when asking for multiple ranges," " does your server at %s:%s support range requests?", self.host, self.port, ) except OSError as e: # It's conceivable that the socket is in a bad state here # (including some test cases) and in this case, it doesn't need # cleaning anymore, so no need to fail, we just get rid of the # socket and let callers reconnect if len(e.args) == 0 or e.args[0] not in ( errno.ECONNRESET, errno.ECONNABORTED, ): raise self.close() self._response = None # Preserve our preciousss sock = self.sock self.sock = None # Let http.client.HTTPConnection do its housekeeping self.close() # Restore our preciousss self.sock = sock def _wrap_socket_for_reporting(self, sock): """Wrap the socket for activity reporting. Replaces the raw socket with a _ReportingSocket that tracks network activity through the configured callback function. Args: sock: The raw socket object to wrap. """ self.sock = _ReportingSocket(sock, self._report_activity) class HTTPConnection(AbstractHTTPConnection, http.client.HTTPConnection): # type: ignore """HTTP connection with activity reporting and connection management. Extends the standard HTTPConnection with activity reporting capabilities and improved connection management. """ # XXX: Needs refactoring at the caller level. def __init__( self, host, port=None, proxied_host=None, report_activity=None, ca_certs=None ): """Initialize an HTTP connection with activity reporting. Args: host: The hostname or IP address to connect to. port: The port number (optional, defaults to 80). proxied_host: The original host if connecting through a proxy. report_activity: Optional callback to report network activity. ca_certs: Certificate authority certificates (ignored for HTTP). """ AbstractHTTPConnection.__init__(self, report_activity=report_activity) http.client.HTTPConnection.__init__(self, host, port) self.proxied_host = proxied_host # ca_certs is ignored, it's only relevant for https def connect(self): """Establish the HTTP connection with activity reporting. Connects to the server and wraps the socket for activity reporting if debug logging is enabled. """ if debug_logger.isEnabledFor(logging.DEBUG): self._mutter_connect() http.client.HTTPConnection.connect(self) self._wrap_socket_for_reporting(self.sock) class HTTPSConnection(AbstractHTTPConnection, http.client.HTTPSConnection): # type: ignore """HTTPS connection with SSL/TLS support, activity reporting, and connection management. Extends the standard HTTPSConnection with activity reporting capabilities, improved connection management, and SSL certificate handling. """ def __init__( self, host, port=None, key_file=None, cert_file=None, proxied_host=None, report_activity=None, ca_certs=None, ): """Initialize an HTTPS connection with SSL support and activity reporting. Args: host: The hostname or IP address to connect to. port: The port number (optional, defaults to 443). key_file: Path to client private key file for SSL authentication. cert_file: Path to client certificate file for SSL authentication. proxied_host: The original host if connecting through a proxy. report_activity: Optional callback to report network activity. ca_certs: Path to CA certificates file for SSL verification. """ AbstractHTTPConnection.__init__(self, report_activity=report_activity) http.client.HTTPSConnection.__init__(self, host=host, port=port) self.key_file = key_file self.cert_file = cert_file self.proxied_host = proxied_host self.ca_certs = ca_certs def connect(self): """Establish the HTTPS connection with SSL and activity reporting. Connects to the server, wraps the socket for activity reporting, and establishes SSL connection if not using a proxy. """ if debug_logger.isEnabledFor(logging.DEBUG): self._mutter_connect() http.client.HTTPConnection.connect(self) self._wrap_socket_for_reporting(self.sock) if self.proxied_host is None: self.connect_to_origin() def connect_to_origin(self): """Establish SSL connection to the origin server. Sets up SSL context with certificate verification, client certificates, and appropriate SSL settings based on configuration. Handles both direct connections and connections through proxies. Raises: ssl.SSLError: If SSL connection or certificate verification fails. """ from dromedary import http as _mod_http cert_reqs = _mod_http.ssl_cert_reqs() if self.proxied_host is not None: host = self.proxied_host.split(":", 1)[0] else: host = self.host if cert_reqs == ssl.CERT_NONE: logger.warning("Not checking SSL certificate for %s", host) ca_certs = None else: if self.ca_certs is None: ca_certs = _mod_http.ssl_ca_certs() else: ca_certs = self.ca_certs if ca_certs is None: logger.warning( "No valid trusted SSL CA certificates file set. See " "'brz help ssl.ca_certs' for more information on setting " "trusted CAs." ) try: ssl_context = ssl.create_default_context( purpose=ssl.Purpose.SERVER_AUTH, cafile=ca_certs ) ssl_context.check_hostname = cert_reqs != ssl.CERT_NONE if self.cert_file: ssl_context.load_cert_chain( keyfile=self.key_file, certfile=self.cert_file ) ssl_context.verify_mode = cert_reqs ssl_sock = ssl_context.wrap_socket(self.sock, server_hostname=host) except ssl.SSLError: logger.info( "\n" "See `brz help ssl.ca_certs` for how to specify trusted CA" "certificates.\n" "Pass -Ossl.cert_reqs=none to disable certificate " "verification entirely.\n" ) raise # Wrap the ssl socket before anybody use it self._wrap_socket_for_reporting(ssl_sock) class Request(urllib.request.Request): """A custom Request object. urllib.request determines the request method heuristically (based on the presence or absence of data). We set the method statically. The Request object tracks: - the connection the request will be made on. - the authentication parameters needed to preventively set the authentication header once a first authentication have been made. """ def __init__( self, method, url, data=None, headers=None, origin_req_host=None, unverifiable=False, connection=None, parent=None, ): """Initialize a custom HTTP request object. Args: method: HTTP method (GET, POST, PUT, etc.). url: The URL to request. data: Request body data (optional). headers: Dictionary of HTTP headers (optional). origin_req_host: Original request host for redirect tracking. unverifiable: Whether the request is unverifiable. connection: HTTP connection object to use. parent: Parent request for redirect chains. """ if headers is None: headers = {} urllib.request.Request.__init__( self, url, data, headers, origin_req_host, unverifiable ) self.method = method self.connection = connection # To handle redirections self.parent = parent self.redirected_to = None # Unless told otherwise, redirections are not followed self.follow_redirections = False # auth and proxy_auth are dicts containing, at least # (scheme, host, port, realm, user, password, protocol, path). # The dict entries are mostly handled by the AuthHandler. # Some authentication schemes may add more entries. self.auth = {} self.proxy_auth = {} self.proxied_host = None def get_method(self): """Get the HTTP method for this request. Returns: The HTTP method string (GET, POST, etc.). """ return self.method def set_proxy(self, proxy, type): """Set the proxy and remember the proxied host.""" host, port = splitport(self.host) if port is None: # We need to set the default port ourselves way before it gets set # in the HTTP[S]Connection object at build time. conn_class = HTTPSConnection if self.type == "https" else HTTPConnection port = conn_class.default_port self.proxied_host = f"{host}:{port}" urllib.request.Request.set_proxy(self, proxy, type) # When urllib.request makes a https request with our wrapper code and a proxy, # it sets Host to the https proxy, not the host we want to talk to. # I'm fairly sure this is our fault, but what is the cause is an open # question. -- Robert Collins May 8 2010. self.add_unredirected_header("Host", self.proxied_host) class _ConnectRequest(Request): def __init__(self, request): """Constructor. :param request: the first request sent to the proxied host, already processed by the opener (i.e. proxied_host is already set). """ # We give a fake url and redefine selector or urllib.request will be # confused Request.__init__( self, "CONNECT", request.get_full_url(), connection=request.connection ) if request.proxied_host is None: raise AssertionError() self.proxied_host = request.proxied_host def get_selector(self): return self.proxied_host def set_selector(self, selector): self.proxied_host = selector selector = property(get_selector, set_selector) # type: ignore def set_proxy(self, proxy, type): """Set the proxy without remembering the proxied host. We already know the proxied host by definition, the CONNECT request occurs only when the connection goes through a proxy. The usual processing (masquerade the request so that the connection is done to the proxy while the request is targeted at another host) does not apply here. In fact, the connection is already established with proxy and we just want to enable the SSL tunneling. """ urllib.request.Request.set_proxy(self, proxy, type) class ConnectionHandler(urllib.request.BaseHandler): """Provides connection-sharing by pre-processing requests. urllib.request provides no way to access the HTTPConnection object internally used. But we need it in order to achieve connection sharing. So, we add it to the request just before it is processed, and then we override the do_open method for http[s] requests in AbstractHTTPHandler. """ handler_order = 1000 # after all pre-processings def __init__(self, report_activity=None, ca_certs=None): """Initialize the connection handler. Args: report_activity: Optional callback to report network activity. ca_certs: Path to CA certificates file for SSL verification. """ self._report_activity = report_activity self.ca_certs = ca_certs def create_connection(self, request, http_connection_class): """Create a new HTTP connection for the request. Args: request: The HTTP request object. http_connection_class: The connection class to instantiate. Returns: A new HTTP connection object. Raises: urlutils.InvalidURL: If the request has no host or invalid URL. """ host = request.host if not host: # Just a bit of paranoia here, this should have been # handled in the higher levels raise urlutils.InvalidURL(request.get_full_url(), "no host given.") # We create a connection (but it will not connect until the first # request is made) try: connection = http_connection_class( host, proxied_host=request.proxied_host, report_activity=self._report_activity, ca_certs=self.ca_certs, ) except http.client.InvalidURL as e: # There is only one occurrence of InvalidURL in http.client raise urlutils.InvalidURL( request.get_full_url(), extra="nonnumeric port" ) from e return connection def capture_connection(self, request, http_connection_class): """Capture or inject the request connection. Two cases: - the request have no connection: create a new one, - the request have a connection: this one have been used already, let's capture it, so that we can give it to another transport to be reused. We don't do that ourselves: the Transport object get the connection from a first request and then propagate it, from request to request or to cloned transports. """ connection = request.connection if connection is None: # Create a new one connection = self.create_connection(request, http_connection_class) request.connection = connection # All connections will pass here, propagate debug level connection.set_debuglevel(DEBUG) return request def http_request(self, request): """Handle HTTP requests by capturing/creating connections. Args: request: The HTTP request object. Returns: The request object with connection attached. """ return self.capture_connection(request, HTTPConnection) def https_request(self, request): """Handle HTTPS requests by capturing/creating connections. Args: request: The HTTPS request object. Returns: The request object with connection attached. """ return self.capture_connection(request, HTTPSConnection) class AbstractHTTPHandler(urllib.request.AbstractHTTPHandler): """A custom handler for HTTP(S) requests. We overrive urllib.request.AbstractHTTPHandler to get a better control of the connection, the ability to implement new request types and return a response able to cope with persistent connections. """ # We change our order to be before urllib.request HTTP[S]Handlers # and be chosen instead of them (the first http_open called # wins). handler_order = 400 @property def _default_headers(self): return { "Pragma": "no-cache", "Cache-control": "max-age=0", "Connection": "Keep-Alive", "User-agent": default_user_agent(), "Accept": "*/*", } def __init__(self): """Initialize the abstract HTTP handler. Sets up the HTTP handler with the configured debug level. """ urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=DEBUG) def http_request(self, request): """Common headers setting.""" for name, value in self._default_headers.items(): if name not in request.headers: request.headers[name] = value # FIXME: We may have to add the Content-Length header if # we have data to send. return request def retry_or_raise(self, http_class, request, first_try): """Retry the request (once) or raise the exception. urllib.request raises exception of application level kind, we just have to translate them. http.client can raise exceptions of transport level (badly formatted dialog, loss of connexion or socket level problems). In that case we should issue the request again (http.client will close and reopen a new connection if needed). """ # When an exception occurs, we give back the original # Traceback or the bugs are hard to diagnose. exc_type, exc_val, exc_tb = sys.exc_info() if exc_type == socket.gaierror: # No need to retry, that will not help origin_req_host = request.origin_req_host raise ConnectionError( f"Couldn't resolve host '{origin_req_host}'", orig_error=exc_val ) elif isinstance(exc_val, http.client.ImproperConnectionState): # The http.client pipeline is in incorrect state, it's a bug in our # implementation. raise exc_val.with_traceback(exc_tb) else: if first_try: if self._debuglevel >= 2: print(f"Received exception: [{exc_val!r}]") print(f" On connection: [{request.connection!r}]") method = request.get_method() url = request.get_full_url() print(f" Will retry, {method} {url!r}") request.connection.close() response = self.do_open(http_class, request, False) else: if self._debuglevel >= 2: print(f"Received second exception: [{exc_val!r}]") print(f" On connection: [{request.connection!r}]") if exc_type in (http.client.BadStatusLine, http.client.UnknownProtocol): # http.client.BadStatusLine and # http.client.UnknownProtocol indicates that a # bogus server was encountered or a bad # connection (i.e. transient errors) is # experimented, we have already retried once # for that request so we raise the exception. my_exception = InvalidHttpResponse( request.get_full_url(), "Bad status line received", orig_error=exc_val, ) elif ( isinstance(exc_val, socket.error) and len(exc_val.args) and exc_val.args[0] in (errno.ECONNRESET, 10053, 10054) ): # 10053 == WSAECONNABORTED # 10054 == WSAECONNRESET raise ConnectionResetError("Connection lost while sending request.") else: # All other exception are considered connection related. # socket errors generally occurs for reasons # far outside our scope, so closing the # connection and retrying is the best we can # do. selector = request.selector my_exception = ConnectionError( f"while sending {request.get_method()} {selector}:" ) if self._debuglevel >= 2: print(f"On connection: [{request.connection!r}]") method = request.get_method() url = request.get_full_url() print(f" Failed again, {method} {url!r}") print(f" Will raise: [{my_exception!r}]") raise my_exception.with_traceback(exc_tb) return response def do_open(self, http_class, request, first_try=True): """See urllib.request.AbstractHTTPHandler.do_open for the general idea. The request will be retried once if it fails. """ connection = request.connection if connection is None: raise AssertionError("Cannot process a request without a connection") # Get all the headers headers = {} headers.update(request.header_items()) headers.update(request.unredirected_hdrs) # Some servers or proxies will choke on headers not properly # cased. http.client/urllib/urllib.request all use capitalize to get canonical # header names, but only python2.5 urllib.request use title() to fix them just # before sending the request. And not all versions of python 2.5 do # that. Since we replace urllib.request.AbstractHTTPHandler.do_open we do it # ourself below. headers = {name.title(): val for name, val in headers.items()} try: method = request.get_method() url = request.selector connection._send_request( method, url, # FIXME: implements 100-continue # None, # We don't send the body yet request.data, headers, encode_chunked=(headers.get("Transfer-Encoding") == "chunked"), ) if debug_logger.isEnabledFor(logging.DEBUG): logger.debug("> %s %s", method, url) hdrs = [] for k, v in headers.items(): # People are often told to paste -Dhttp output to help # debug. Don't compromise credentials. if k in ("Authorization", "Proxy-Authorization"): v = "" hdrs.append(f"{k}: {v}") logger.debug("> " + "\n> ".join(hdrs) + "\n") if self._debuglevel >= 1: print( f"Request sent: [{request!r}] from ({request.connection.sock.getsockname()})" ) response = connection.getresponse() convert_to_addinfourl = True except (ssl.SSLError, ssl.CertificateError): # Something is wrong with either the certificate or the hostname, # re-trying won't help raise except ( socket.gaierror, http.client.BadStatusLine, http.client.UnknownProtocol, OSError, http.client.HTTPException, ): response = self.retry_or_raise(http_class, request, first_try) convert_to_addinfourl = False response.msg = response.reason return response # FIXME: HTTPConnection does not fully support 100-continue (the # server responses are just ignored) # if code == 100: # mutter('Will send the body') # # We can send the body now # body = request.data # if body is None: # raise URLError("No data given") # connection.send(body) # response = connection.getresponse() if self._debuglevel >= 2: print(f"Receives response: {response!r}") print(f" For: {request.get_method()!r}({request.get_full_url()!r})") if convert_to_addinfourl: # Shamelessly copied from urllib.request req = request r = response r.recv = r.read fp = socket._fileobject(r, bufsize=65536) resp = urllib.request.addinfourl(fp, r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason resp.version = r.version if self._debuglevel >= 2: print("Create addinfourl: {!r}".format(resp)) print(f" For: {request.get_method()!r}({request.get_full_url()!r})") if debug_logger.isEnabledFor(logging.DEBUG): version = "HTTP/%d.%d" try: version = version % (resp.version / 10, resp.version % 10) except BaseException: version = f"HTTP/{resp.version!r}" logger.debug("< %s %s %s", version, resp.code, resp.msg) # Use the raw header lines instead of treating resp.info() as a # dict since we may miss duplicated headers otherwise. hdrs = [h.rstrip("\r\n") for h in resp.info().headers] logger.debug("< " + "\n< ".join(hdrs) + "\n") else: resp = response return resp class HTTPHandler(AbstractHTTPHandler): """A custom handler that just thunks into HTTPConnection.""" def http_open(self, request): """Open an HTTP connection for the request. Args: request: The HTTP request object. Returns: Response from the HTTP connection. """ return self.do_open(HTTPConnection, request) class HTTPSHandler(AbstractHTTPHandler): """A custom handler that just thunks into HTTPSConnection.""" https_request = AbstractHTTPHandler.http_request def https_open(self, request): """Open an HTTPS connection for the request. Handles HTTPS connections including proxy CONNECT tunneling when needed. Args: request: The HTTPS request object. Returns: Response from the HTTPS connection. """ connection = request.connection if ( connection.sock is None and connection.proxied_host is not None and request.get_method() != "CONNECT" ): # Don't loop # FIXME: We need a gazillion connection tests here, but we still # miss a https server :-( : # - with and without proxy # - with and without certificate # - with self-signed certificate # - with and without authentication # - with good and bad credentials (especially the proxy auth around # CONNECT) # - with basic and digest schemes # - reconnection on errors # - connection persistence behaviour (including reconnection) # We are about to connect for the first time via a proxy, we must # issue a CONNECT request first to establish the encrypted link connect = _ConnectRequest(request) response = self.parent.open(connect) if response.code != 200: raise ConnectionError( f"Can't connect to {connect.proxied_host} via proxy {self.host}" ) # Housekeeping connection.cleanup_pipe() # Establish the connection encryption connection.connect_to_origin() # Propagate the connection to the original request request.connection = connection return self.do_open(HTTPSConnection, request) class HTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """Handles redirect requests. We have to implement our own scheme because we use a specific Request object and because we want to implement a specific policy. """ _debuglevel = DEBUG # RFC2616 says that only read requests should be redirected # without interacting with the user. But Breezy uses some # shortcuts to optimize against roundtrips which can leads to # write requests being issued before read requests of # containing dirs can be redirected. So we redirect write # requests in the same way which seems to respect the spirit # of the RFC if not its letter. def redirect_request(self, req, fp, code, msg, headers, newurl): """See urllib.request.HTTPRedirectHandler.redirect_request.""" # We would have preferred to update the request instead # of creating a new one, but the urllib.request.Request object # has a too complicated creation process to provide a # simple enough equivalent update process. Instead, when # redirecting, we only update the following request in # the redirect chain with a reference to the parent # request . # Some codes make no sense in our context and are treated # as errors: # 300: Multiple choices for different representations of # the URI. Using that mechanisn with Breezy will violate the # protocol neutrality of Transport. # 304: Not modified (SHOULD only occurs with conditional # GETs which are not used by our implementation) # 305: Use proxy. I can't imagine this one occurring in # our context-- vila/20060909 # 306: Unused (if the RFC says so...) # If the code is 302 and the request is HEAD, some may # think that it is a sufficent hint that the file exists # and that we MAY avoid following the redirections. But # if we want to be sure, we MUST follow them. origin_req_host = req.origin_req_host if code in (301, 302, 303, 307, 308): return Request( req.get_method(), newurl, headers=req.headers, origin_req_host=origin_req_host, unverifiable=True, # TODO: It will be nice to be able to # detect virtual hosts sharing the same # IP address, that will allow us to # share the same connection... connection=None, parent=req, ) else: raise urllib.request.HTTPError(req.get_full_url(), code, msg, headers, fp) def http_error_302(self, req, fp, code, msg, headers): """Requests the redirected to URI. Copied from urllib.request to be able to clean the pipe of the associated connection, *before* issuing the redirected request but *after* having eventually raised an error. """ # Some servers (incorrectly) return multiple Location headers # (so probably same goes for URI). Use first header. # TODO: Once we get rid of addinfourl objects, the # following will need to be updated to use correct case # for headers. if "location" in headers: newurl = headers.get("location") elif "uri" in headers: newurl = headers.get("uri") else: return newurl = urljoin(req.get_full_url(), newurl) if self._debuglevel >= 1: print(f"Redirected to: {newurl} (followed: {req.follow_redirections!r})") if req.follow_redirections is False: req.redirected_to = newurl return fp # This call succeeds or raise an error. urllib.request returns # if redirect_request returns None, but our # redirect_request never returns None. redirected_req = self.redirect_request(req, fp, code, msg, headers, newurl) # loop detection # .redirect_dict has a key url if url was previously visited. if hasattr(req, "redirect_dict"): visited = redirected_req.redirect_dict = req.redirect_dict if ( visited.get(newurl, 0) >= self.max_repeats or len(visited) >= self.max_redirections ): raise urllib.request.HTTPError( req.get_full_url(), code, self.inf_msg + msg, headers, fp ) else: visited = redirected_req.redirect_dict = req.redirect_dict = {} visited[newurl] = visited.get(newurl, 0) + 1 # We can close the fp now that we are sure that we won't # use it with HTTPError. fp.close() # We have all we need already in the response req.connection.cleanup_pipe() return self.parent.open(redirected_req) http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302 class ProxyHandler(urllib.request.ProxyHandler): """Handles proxy setting. Copied and modified from urllib.request to be able to modify the request during the request pre-processing instead of modifying it at _open time. As we capture (or create) the connection object during request processing, _open time was too late. The main task is to modify the request so that the connection is done to the proxy while the request still refers to the destination host. Note: the proxy handling *may* modify the protocol used; the request may be against an https server proxied through an http proxy. So, https_request will be called, but later it's really http_open that will be called. This explains why we don't have to call self.parent.open as the urllib.request did. """ # Proxies must be in front handler_order = 100 _debuglevel = DEBUG def __init__(self, proxies=None): """Initialize ProxyHandler with proxy configuration. Args: proxies: Dictionary of proxy settings by protocol. """ urllib.request.ProxyHandler.__init__(self, proxies) # First, let's get rid of urllib.request implementation for type, proxy in self.proxies.items(): if self._debuglevel >= 3: print(f"Will unbind {type}_open for {proxy!r}") delattr(self, f"{type}_open") def bind_scheme_request(proxy, scheme): if proxy is None: return scheme_request = scheme + "_request" if self._debuglevel >= 3: print(f"Will bind {scheme_request} for {proxy!r}") setattr( self, scheme_request, lambda request: self.set_proxy(request, scheme) ) # We are interested only by the http[s] proxies http_proxy = self.get_proxy_env_var("http") bind_scheme_request(http_proxy, "http") https_proxy = self.get_proxy_env_var("https") bind_scheme_request(https_proxy, "https") def get_proxy_env_var(self, name, default_to="all"): """Get a proxy env var. Note that we indirectly rely on urllib.getproxies_environment taking into account the uppercased values for proxy variables. """ try: return self.proxies[name.lower()] except KeyError: if default_to is not None: # Try to get the alternate environment variable try: return self.proxies[default_to] except KeyError: pass return None def proxy_bypass(self, host): """Check if host should be proxied or not. :returns: True to skip the proxy, False otherwise. """ no_proxy = self.get_proxy_env_var("no", default_to=None) bypass = self.evaluate_proxy_bypass(host, no_proxy) if bypass is None: # Nevertheless, there are platform-specific ways to # ignore proxies... return urllib.request.proxy_bypass(host) else: return bypass def evaluate_proxy_bypass(self, host, no_proxy): """Check the host against a comma-separated no_proxy list as a string. :param host: ``host:port`` being requested :param no_proxy: comma-separated list of hosts to access directly. :returns: True to skip the proxy, False not to, or None to leave it to urllib. """ if no_proxy is None: # All hosts are proxied return False hhost, hport = splitport(host) # Does host match any of the domains mentioned in # no_proxy ? The rules about what is authorized in no_proxy # are fuzzy (to say the least). We try to allow most # commonly seen values. for domain in no_proxy.split(","): domain = domain.strip() if domain == "": continue dhost, dport = splitport(domain) if hport == dport or dport is None: # Protect glob chars dhost = dhost.replace(".", r"\.") dhost = dhost.replace("*", r".*") dhost = dhost.replace("?", r".") if re.match(dhost, hhost, re.IGNORECASE): return True # Nothing explicitly avoid the host return None def set_proxy(self, request, type): """Set proxy for request if not bypassed. Args: request: HTTP request object. type: Protocol type (http, https, etc.). """ host = request.host if self.proxy_bypass(host): return request proxy = self.get_proxy_env_var(type) if self._debuglevel >= 3: print(f"set_proxy {type}_request for {proxy!r}") # FIXME: python 2.5 urlparse provides a better _parse_proxy which can # grok user:password@host:port as well as # http://user:password@host:port parsed_url = _mod_dromedary.ConnectedTransport._split_url(proxy) if not parsed_url.host: raise urlutils.InvalidURL(proxy, "No host component") if request.proxy_auth == {}: # No proxy auth parameter are available, we are handling the first # proxied request, intialize. scheme (the authentication scheme) # and realm will be set by the AuthHandler request.proxy_auth = { "host": parsed_url.host, "port": parsed_url.port, "user": parsed_url.user, "password": parsed_url.password, "protocol": parsed_url.scheme, # We ignore path since we connect to a proxy "path": None, } if parsed_url.port is None: phost = parsed_url.host else: phost = parsed_url.host + ":%d" % parsed_url.port request.set_proxy(phost, type) if self._debuglevel >= 3: print(f"set_proxy: proxy set to {type}://{phost}") return request class AbstractAuthHandler(urllib.request.BaseHandler): """A custom abstract authentication handler for all http authentications. Provides the meat to handle authentication errors and preventively set authentication headers after the first successful authentication. This can be used for http and proxy, as well as for basic, negotiate and digest authentications. This provides an unified interface for all authentication handlers (urllib.request provides far too many with different policies). The interaction between this handler and the urllib.request framework is not obvious, it works as follow: opener.open(request) is called: - that may trigger http_request which will add an authentication header (self.build_header) if enough info is available. - the request is sent to the server, - if an authentication error is received self.auth_required is called, we acquire the authentication info in the error headers and call self.auth_match to check that we are able to try the authentication and complete the authentication parameters, - we call parent.open(request), that may trigger http_request and will add a header (self.build_header), but here we have all the required info (keep in mind that the request and authentication used in the recursive calls are really (and must be) the *same* objects). - if the call returns a response, the authentication have been successful and the request authentication parameters have been updated. """ scheme: str """The scheme as it appears in the server header (lower cased)""" _max_retry = 3 """We don't want to retry authenticating endlessly""" requires_username = True """Whether the auth mechanism requires a username.""" # The following attributes should be defined by daughter # classes: # - auth_required_header: the header received from the server # - auth_header: the header sent in the request def __init__(self): """Initialize the authentication handler.""" # We want to know when we enter into an try/fail cycle of # authentications so we initialize to None to indicate that we aren't # in such a cycle by default. self._retry_count = None def _parse_auth_header(self, server_header): """Parse the authentication header. :param server_header: The value of the header sent by the server describing the authenticaion request. :return: A tuple (scheme, remainder) scheme being the first word in the given header (lower cased), remainder may be None. """ try: scheme, remainder = server_header.split(None, 1) except ValueError: scheme = server_header remainder = None return (scheme.lower(), remainder) def update_auth(self, auth, key, value): """Update a value in auth marking the auth as modified if needed.""" old_value = auth.get(key, None) if old_value != value: auth[key] = value auth["modified"] = True def auth_required(self, request, headers): """Retry the request if the auth scheme is ours. :param request: The request needing authentication. :param headers: The headers for the authentication error response. :return: None or the response for the authenticated request. """ # Don't try to authenticate endlessly if self._retry_count is None: # The retry being recusrsive calls, None identify the first retry self._retry_count = 1 else: self._retry_count += 1 if self._retry_count > self._max_retry: # Let's be ready for next round self._retry_count = None return None server_headers = headers.get_all(self.auth_required_header) if not server_headers: # The http error MUST have the associated # header. This must never happen in production code. logger.debug("%s not found", self.auth_required_header) return None auth = self.get_auth(request) auth["modified"] = False # Put some common info in auth if the caller didn't if auth.get("path", None) is None: parsed_url = urlutils.URL.from_string(request.get_full_url()) self.update_auth(auth, "protocol", parsed_url.scheme) self.update_auth(auth, "host", parsed_url.host) self.update_auth(auth, "port", parsed_url.port) self.update_auth(auth, "path", parsed_url.path) # FIXME: the auth handler should be selected at a single place instead # of letting all handlers try to match all headers, but the current # design doesn't allow a simple implementation. for server_header in server_headers: # Several schemes can be proposed by the server, try to match each # one in turn matching_handler = self.auth_match(server_header, auth) if matching_handler: # auth_match may have modified auth (by adding the # password or changing the realm, for example) if ( request.get_header(self.auth_header, None) is not None and not auth["modified"] ): # We already tried that, give up return None # Only the most secure scheme proposed by the server should be # used, since the handlers use 'handler_order' to describe that # property, the first handler tried takes precedence, the # others should not attempt to authenticate if the best one # failed. best_scheme = auth.get("best_scheme", None) if best_scheme is None: # At that point, if current handler should doesn't succeed # the credentials are wrong (or incomplete), but we know # that the associated scheme should be used. best_scheme = auth["best_scheme"] = self.scheme if best_scheme != self.scheme: continue if self.requires_username and auth.get("user", None) is None: # Without a known user, we can't authenticate return None # Housekeeping request.connection.cleanup_pipe() # Retry the request with an authentication header added response = self.parent.open(request) if response: self.auth_successful(request, response) return response # We are not qualified to handle the authentication. # Note: the authentication error handling will try all # available handlers. If one of them authenticates # successfully, a response will be returned. If none of # them succeeds, None will be returned and the error # handler will raise the 401 'Unauthorized' or the 407 # 'Proxy Authentication Required' error. return None def add_auth_header(self, request, header): """Add the authentication header to the request.""" request.add_unredirected_header(self.auth_header, header) def auth_match(self, header, auth): """Check that we are able to handle that authentication scheme. The request authentication parameters may need to be updated with info from the server. Some of these parameters, when combined, are considered to be the authentication key, if one of them change the authentication result may change. 'user' and 'password' are exampls, but some auth schemes may have others (digest's nonce is an example, digest's nonce_count is a *counter-example*). Such parameters must be updated by using the update_auth() method. :param header: The authentication header sent by the server. :param auth: The auth parameters already known. They may be updated. :returns: True if we can try to handle the authentication. """ raise NotImplementedError(self.auth_match) def build_auth_header(self, auth, request): """Build the value of the header used to authenticate. :param auth: The auth parameters needed to build the header. :param request: The request needing authentication. :return: None or header. """ raise NotImplementedError(self.build_auth_header) def auth_successful(self, request, response): """The authentification was successful for the request. Additional infos may be available in the response. :param request: The succesfully authenticated request. :param response: The server response (may contain auth info). """ # It may happen that we need to reconnect later, let's be ready self._retry_count = None def get_user_password(self, auth): """Ask user for a password if none is already available. :param auth: authentication info gathered so far (from the initial url and then during dialog with the server). """ from dromedary import _ui from dromedary.http import get_credentials user = auth.get("user", None) password = auth.get("password", None) realm = auth["realm"] port = auth.get("port", None) if user is None or password is None: looked_up_user, looked_up_password = get_credentials( auth["protocol"], auth["host"], port=port, path=auth["path"], realm=realm, ) if user is None: user = looked_up_user if password is None: password = looked_up_password if user is None: # Prompt for the username if we still don't have one. prompt = self.build_username_prompt(auth) user = _ui.get_username(prompt, **auth) auth["user"] = user if user is not None and password is None: # Prompt for the password if we still don't have one. prompt = self.build_password_prompt(auth) password = _ui.get_password(prompt, **auth) return user, password def _build_password_prompt(self, auth): """Build a prompt taking the protocol used into account. The AuthHandler is used by http and https, we want that information in the prompt, so we build the prompt from the authentication dict which contains all the needed parts. Also, http and proxy AuthHandlers present different prompts to the user. The daughter classes should implements a public build_password_prompt using this method. """ prompt = f"{auth['protocol'].upper()}" + " %(user)s@%(host)s" if auth.get("port") is not None: prompt += ":%(port)d" realm = auth["realm"] if realm is not None: prompt += f", Realm: '{realm}'" prompt += " password" return prompt def _build_username_prompt(self, auth): """Build a prompt taking the protocol used into account. The AuthHandler is used by http and https, we want that information in the prompt, so we build the prompt from the authentication dict which contains all the needed parts. Also, http and proxy AuthHandlers present different prompts to the user. The daughter classes should implements a public build_username_prompt using this method. """ prompt = f"{auth['protocol'].upper()}" + " %(host)s" if auth.get("port") is not None: prompt += ":%(port)d" realm = auth["realm"] if realm is not None: prompt += f", Realm: '{realm}'" prompt += " username" return prompt def http_request(self, request): """Insert an authentication header if information is available.""" auth = self.get_auth(request) if self.auth_params_reusable(auth): self.add_auth_header(request, self.build_auth_header(auth, request)) return request https_request = http_request # FIXME: Need test class NegotiateAuthHandler(AbstractAuthHandler): """A authentication handler that handles WWW-Authenticate: Negotiate. At the moment this handler supports just Kerberos. In the future, NTLM support may also be added. """ scheme = "negotiate" handler_order = 480 requires_username = False def auth_match(self, header, auth): """Check if authentication matches the header. Args: header: Authentication header from server. auth: Authentication dictionary. Returns: bool: True if authentication matches. """ scheme, _raw_auth = self._parse_auth_header(header) if scheme != self.scheme: return False self.update_auth(auth, "scheme", scheme) resp = self._auth_match_kerberos(auth) if resp is None: return False # Optionally should try to authenticate using NTLM here self.update_auth(auth, "negotiate_response", resp) return True def _auth_match_kerberos(self, auth): """Try to create a GSSAPI response for authenticating against a host.""" global kerberos, checked_kerberos if kerberos is None and not checked_kerberos: try: import kerberos except ModuleNotFoundError: kerberos = None checked_kerberos = True if kerberos is None: return None ret, vc = kerberos.authGSSClientInit(f"HTTP@{auth['host']}") if ret < 1: logger.warning( "Unable to create GSSAPI context for %s: %d", auth["host"], ret ) return None ret = kerberos.authGSSClientStep(vc, "") if ret < 0: logger.debug("authGSSClientStep failed: %d", ret) return None return kerberos.authGSSClientResponse(vc) def build_auth_header(self, auth, request): """Build authentication header for request. Args: auth: Authentication dictionary. request: Request object. Returns: str: Authentication header value. """ return f"Negotiate {auth['negotiate_response']}" def auth_params_reusable(self, auth): """Check if authentication parameters are reusable. Args: auth: Authentication dictionary. Returns: bool: True if parameters can be reused. """ # If the auth scheme is known, it means a previous # authentication was successful, all information is # available, no further checks are needed. return ( auth.get("scheme", None) == "negotiate" and auth.get("negotiate_response", None) is not None ) class BasicAuthHandler(AbstractAuthHandler): """A custom basic authentication handler.""" scheme = "basic" handler_order = 500 auth_regexp = re.compile('realm="([^"]*)"', re.I) def build_auth_header(self, auth, request): """Build basic authentication header. Args: auth: Authentication dictionary with user and password. request: Request object. Returns: str: Basic authentication header value. """ raw = f"{auth['user']}:{auth['password']}" auth_header = "Basic " + base64.b64encode(raw.encode("utf-8")).decode("ascii") return auth_header def extract_realm(self, header_value): """Extract realm from authentication header. Args: header_value: Authentication header value. Returns: tuple: (match object, realm string). """ match = self.auth_regexp.search(header_value) realm = None if match: realm = match.group(1) return match, realm def auth_match(self, header, auth): """Check if basic authentication matches the header. Args: header: Authentication header from server. auth: Authentication dictionary. Returns: bool: True if authentication matches. """ scheme, raw_auth = self._parse_auth_header(header) if scheme != self.scheme: return False match, realm = self.extract_realm(raw_auth) if match: # Put useful info into auth self.update_auth(auth, "scheme", scheme) self.update_auth(auth, "realm", realm) if auth.get("user", None) is None or auth.get("password", None) is None: user, password = self.get_user_password(auth) self.update_auth(auth, "user", user) self.update_auth(auth, "password", password) return match is not None def auth_params_reusable(self, auth): """Check if basic authentication parameters are reusable. Args: auth: Authentication dictionary. Returns: bool: True if parameters can be reused. """ # If the auth scheme is known, it means a previous # authentication was successful, all information is # available, no further checks are needed. return auth.get("scheme", None) == "basic" def get_digest_algorithm_impls(algorithm): """Get digest algorithm implementations for HTTP authentication. Creates hash function implementations (H) and key derivation function (KD) for the specified digest algorithm used in HTTP digest authentication. Args: algorithm: The digest algorithm name (e.g., 'MD5', 'SHA'). Returns: A tuple of (H, KD) where: - H: Hash function that takes bytes and returns hex digest - KD: Key derivation function that takes secret and data strings Both functions return None if the algorithm is not supported. """ H = None KD = None if algorithm == "MD5": def H(x): return hashlib.md5(x).hexdigest() # noqa: S324 elif algorithm == "SHA": def H(x): return hashlib.sha1(x).hexdigest() # noqa: S324 if H is not None: def KD(secret, data): return H(f"{secret}:{data}".encode()) return H, KD def get_new_cnonce(nonce, nonce_count): """Generate a new client nonce for HTTP digest authentication. Creates a unique client nonce by combining the server nonce, nonce count, current time, and random characters, then hashing the result. Args: nonce: The server-provided nonce string. nonce_count: The number of times this nonce has been used. Returns: A 16-character hexadecimal string to use as client nonce. """ raw = "%s:%d:%s:%s" % (nonce, nonce_count, time.ctime(), rand_chars(8)) return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:16] # noqa: S324 class DigestAuthHandler(AbstractAuthHandler): """A custom digest authentication handler.""" scheme = "digest" # Before basic as digest is a bit more secure and should be preferred handler_order = 490 def auth_params_reusable(self, auth): """Check if authentication parameters are reusable. Args: auth: Authentication dictionary containing auth parameters. Returns: bool: True if the auth scheme is 'digest', indicating previous successful authentication with reusable parameters. """ # If the auth scheme is known, it means a previous # authentication was successful, all information is # available, no further checks are needed. return auth.get("scheme", None) == "digest" def auth_match(self, header, auth): """Check if authentication header matches this handler's scheme. Args: header: WWW-Authenticate header value. auth: Authentication dictionary to update with parsed values. Returns: bool: True if the header scheme matches 'digest' and can be handled. """ scheme, raw_auth = self._parse_auth_header(header) if scheme != self.scheme: return False # Put the requested authentication info into a dict req_auth = urllib.request.parse_keqv_list( urllib.request.parse_http_list(raw_auth) ) # Check that we can handle that authentication qop = req_auth.get("qop", None) if qop != "auth": # No auth-int so far return False H, _KD = get_digest_algorithm_impls(req_auth.get("algorithm", "MD5")) if H is None: return False realm = req_auth.get("realm", None) # Put useful info into auth self.update_auth(auth, "scheme", scheme) self.update_auth(auth, "realm", realm) if auth.get("user", None) is None or auth.get("password", None) is None: user, password = self.get_user_password(auth) self.update_auth(auth, "user", user) self.update_auth(auth, "password", password) try: if req_auth.get("algorithm", None) is not None: self.update_auth(auth, "algorithm", req_auth.get("algorithm")) nonce = req_auth["nonce"] if auth.get("nonce", None) != nonce: # A new nonce, never used self.update_auth(auth, "nonce_count", 0) self.update_auth(auth, "nonce", nonce) self.update_auth(auth, "qop", qop) auth["opaque"] = req_auth.get("opaque", None) except KeyError: # Some required field is not there return False return True def build_auth_header(self, auth, request): """Build the Authorization header for digest authentication. Args: auth: Authentication dictionary with user credentials and parameters. request: The HTTP request object. Returns: Tuple of (header_name, header_value) for the Authorization header. """ uri = urlparse(request.selector).path A1 = f"{auth['user']}:{auth['realm']}:{auth['password']}".encode() A2 = f"{request.get_method()}:{uri}".encode() nonce = auth["nonce"] qop = auth["qop"] nonce_count = auth["nonce_count"] + 1 ncvalue = f"{nonce_count:08x}" cnonce = get_new_cnonce(nonce, nonce_count) H, KD = get_digest_algorithm_impls(auth.get("algorithm", "MD5")) nonce_data = f"{nonce}:{ncvalue}:{cnonce}:{qop}:{H(A2)}" request_digest = KD(H(A1), nonce_data) header = "Digest " header += 'username="{}", realm="{}", nonce="{}"'.format( auth["user"], auth["realm"], nonce ) header += f', uri="{uri}"' header += f', cnonce="{cnonce}", nc={ncvalue}' header += f', qop="{qop}"' header += f', response="{request_digest}"' # Append the optional fields opaque = auth.get("opaque", None) if opaque: header += f', opaque="{opaque}"' if auth.get("algorithm", None): header += f', algorithm="{auth.get("algorithm")}"' # We have used the nonce once more, update the count auth["nonce_count"] = nonce_count return header class HTTPAuthHandler(AbstractAuthHandler): """Custom http authentication handler. Send the authentication preventively to avoid the roundtrip associated with the 401 error and keep the revelant info in the auth request attribute. """ auth_required_header = "www-authenticate" auth_header = "Authorization" def get_auth(self, request): """Get the auth params from the request.""" return request.auth def set_auth(self, request, auth): """Set the auth params for the request.""" request.auth = auth def build_password_prompt(self, auth): """Build a password prompt for the authentication realm. Args: auth: Authentication dictionary containing realm info. Returns: Formatted password prompt string. """ return self._build_password_prompt(auth) def build_username_prompt(self, auth): """Build a username prompt for the authentication realm. Args: auth: Authentication dictionary containing realm info. Returns: Formatted username prompt string. """ return self._build_username_prompt(auth) def http_error_401(self, req, fp, code, msg, headers): """Handle HTTP 401 Unauthorized error. Args: req: The request that resulted in 401. fp: File pointer to the error response. code: HTTP status code (401). msg: Error message. headers: Response headers. Returns: Result of auth_required processing. """ return self.auth_required(req, headers) class ProxyAuthHandler(AbstractAuthHandler): """Custom proxy authentication handler. Send the authentication preventively to avoid the roundtrip associated with the 407 error and keep the revelant info in the proxy_auth request attribute.. """ auth_required_header = "proxy-authenticate" # FIXME: the correct capitalization is Proxy-Authorization, # but python-2.4 urllib.request.Request insist on using capitalize() # instead of title(). auth_header = "Proxy-authorization" def get_auth(self, request): """Get the auth params from the request.""" return request.proxy_auth def set_auth(self, request, auth): """Set the auth params for the request.""" request.proxy_auth = auth def build_password_prompt(self, auth): """Build a password prompt for proxy authentication. Args: auth: Authentication dictionary containing realm info. Returns: Formatted password prompt string prefixed with 'Proxy '. """ prompt = self._build_password_prompt(auth) prompt = "Proxy " + prompt return prompt def build_username_prompt(self, auth): """Build a username prompt for proxy authentication. Args: auth: Authentication dictionary containing realm info. Returns: Formatted username prompt string prefixed with 'Proxy '. """ prompt = self._build_username_prompt(auth) prompt = "Proxy " + prompt return prompt def http_error_407(self, req, fp, code, msg, headers): """Handle HTTP 407 Proxy Authentication Required error. Args: req: The request that resulted in 407. fp: File pointer to the error response. code: HTTP status code (407). msg: Error message. headers: Response headers. Returns: Result of auth_required processing. """ return self.auth_required(req, headers) class HTTPBasicAuthHandler(BasicAuthHandler, HTTPAuthHandler): """Custom http basic authentication handler.""" class ProxyBasicAuthHandler(BasicAuthHandler, ProxyAuthHandler): """Custom proxy basic authentication handler.""" class HTTPDigestAuthHandler(DigestAuthHandler, HTTPAuthHandler): """Custom http basic authentication handler.""" class ProxyDigestAuthHandler(DigestAuthHandler, ProxyAuthHandler): """Custom proxy basic authentication handler.""" class HTTPNegotiateAuthHandler(NegotiateAuthHandler, HTTPAuthHandler): """Custom http negotiate authentication handler.""" class ProxyNegotiateAuthHandler(NegotiateAuthHandler, ProxyAuthHandler): """Custom proxy negotiate authentication handler.""" class HTTPErrorProcessor(urllib.request.HTTPErrorProcessor): """Process HTTP error responses. We don't really process the errors, quite the contrary instead, we leave our Transport handle them. """ accepted_errors = [ 200, # Ok 201, 202, 204, 206, # Partial content 207, # Multi-Status Response (for webdav) 400, 403, 404, # Not found 405, # Method not allowed 406, # Not Acceptable 409, # Conflict 412, # Precondition failed (for webdav) 416, # Range not satisfiable 422, # Unprocessible entity 501, # Not implemented ] """The error codes the caller will handle. This can be specialized in the request on a case-by case basis, but the common cases are covered here. """ def http_response(self, request, response): """Process HTTP response and handle errors. Args: request: The HTTP request that generated this response. response: The HTTP response object. Returns: The response object, possibly modified by error handling. """ code, msg, hdrs = response.code, response.msg, response.info() if code not in self.accepted_errors: response = self.parent.error("http", request, response, code, msg, hdrs) return response https_response = http_response class HTTPDefaultErrorHandler(urllib.request.HTTPDefaultErrorHandler): """Translate common errors into Breezy Exceptions.""" def http_error_default(self, req, fp, code, msg, hdrs): """Handle default HTTP errors by translating to Breezy exceptions. Args: req: The request that resulted in an error. fp: File pointer to the error response. code: HTTP status code. msg: Error message. hdrs: Response headers. Raises: TransportError: For 403 Forbidden errors. InvalidHttpResponse: For other error codes. """ if code == 403: raise TransportError( "Server refuses to fulfill the request (403 Forbidden) for {}".format( req.get_full_url() ) ) else: raise UnexpectedHttpStatus( req.get_full_url(), code, f"Unable to handle http code: {msg}", headers=hdrs, ) class Opener: """A wrapper around urllib.request.build_opener. Daughter classes can override to build their own specific opener """ # TODO: Provides hooks for daughter classes. def __init__( self, connection=ConnectionHandler, redirect=HTTPRedirectHandler, error=HTTPErrorProcessor, report_activity=None, ca_certs=None, ): """Initialize the opener. Args: connection: Connection handler class to use. redirect: Redirect handler class to use. error: Error processor class to use. report_activity: Optional callback for reporting activity. ca_certs: Path to CA certificates file. """ self._opener = urllib.request.build_opener( connection(report_activity=report_activity, ca_certs=ca_certs), redirect, error, ProxyHandler(), HTTPBasicAuthHandler(), HTTPDigestAuthHandler(), HTTPNegotiateAuthHandler(), ProxyBasicAuthHandler(), ProxyDigestAuthHandler(), ProxyNegotiateAuthHandler(), HTTPHandler, HTTPSHandler, HTTPDefaultErrorHandler, ) self.open = self._opener.open if DEBUG >= 9: # When dealing with handler order, it's easy to mess # things up, the following will help understand which # handler is used, when and for what. import pprint pprint.pprint(self._opener.__dict__) class HttpTransport(ConnectedTransport): """HTTP Client implementations. The protocol can be given as e.g. http+urllib://host/ to use a particular implementation. """ # _unqualified_scheme: "http" or "https" # _scheme: may have "+pycurl", etc # In order to debug we have to issue our traces in sync with # httplib, which use print :( _debuglevel = 0 def __init__(self, base, _from_transport=None, ca_certs=None): """Set the base path where files will be stored.""" proto_match = re.match(r"^(https?)(\+\w+)?://", base) if not proto_match: raise AssertionError(f"not a http url: {base!r}") self._unqualified_scheme = proto_match.group(1) super().__init__(base, _from_transport=_from_transport) self._medium = None # range hint is handled dynamically throughout the life # of the transport object. We start by trying multi-range # requests and if the server returns bogus results, we # retry with single range requests and, finally, we # forget about range if the server really can't # understand. Once acquired, this piece of info is # propagated to clones. if _from_transport is not None: self._range_hint = _from_transport._range_hint self._opener = _from_transport._opener else: self._range_hint = "multi" self._opener = Opener( report_activity=self._report_activity, ca_certs=ca_certs ) def request(self, method, url, fields=None, headers=None, **urlopen_kw): """Make an HTTP request with the specified method. Args: method: HTTP method (GET, POST, etc). url: URL to request. fields: Form fields to encode as URL parameters. headers: Additional HTTP headers. **urlopen_kw: Additional keyword arguments (body, retries). Returns: The HTTP response object. Raises: ValueError: If both body and fields are provided. NotImplementedError: If unknown urlopen_kw arguments are provided. """ body = urlopen_kw.pop("body", None) if fields is not None: data = urlencode(fields).encode() if body is not None: raise ValueError("body and fields are mutually exclusive") else: data = body if headers is None: headers = {} request = Request(method, url, data, headers) request.follow_redirections = urlopen_kw.pop("retries", 0) > 0 if urlopen_kw: raise NotImplementedError(f"unknown arguments: {urlopen_kw.keys()!r}") connection = self._get_connection() if connection is not None: # Give back shared info request.connection = connection (auth, proxy_auth) = self._get_credentials() # Clean the httplib.HTTPConnection pipeline in case the previous # request couldn't do it connection.cleanup_pipe() else: # First request, initialize credentials. # scheme and realm will be set by the _urllib2_wrappers.AuthHandler auth = self._create_auth() # Proxy initialization will be done by the first proxied request proxy_auth = {} # Ensure authentication info is provided request.auth = auth request.proxy_auth = proxy_auth if self._debuglevel > 0: print( "perform: {} base: {}, url: {}".format( request.method, self.base, request.get_full_url() ) ) response = self._opener.open(request) if self._get_connection() is not request.connection: # First connection or reconnection self._set_connection(request.connection, (request.auth, request.proxy_auth)) else: # http may change the credentials while keeping the # connection opened self._update_credentials((request.auth, request.proxy_auth)) code = response.code if request.follow_redirections is False and code in (301, 302, 303, 307, 308): raise RedirectRequested( request.get_full_url(), request.redirected_to, is_permanent=(code in (301, 308)), ) if request.redirected_to is not None: logger.debug( "redirected from: %s to: %s", request.get_full_url(), request.redirected_to, ) class Urllib3LikeResponse: def __init__(self, actual): self._actual = actual self._data = None def getheader(self, name, default=None): if self._actual.headers is None: raise http.client.ResponseNotReady() return self._actual.headers.get(name, default) def getheaders(self): if self._actual.headers is None: raise http.client.ResponseNotReady() return list(self._actual.headers.items()) @property def status(self): return self._actual.code @property def reason(self): return self._actual.reason @property def data(self): if self._data is None: self._data = self._actual.read() return self._data @property def text(self): if self.status == 204: return None from email.message import EmailMessage msg = EmailMessage() msg["content-type"] = self._actual.headers["Content-Type"] charset = msg["content-type"].params.get("charset") if charset: return self.data.decode(charset) else: return self.data.decode() def read(self, amt=None): """Read data from the response. Args: amt: Number of bytes to read. If None, reads all data. Returns: The data read from the response. """ if amt is None and evil_logger.isEnabledFor(logging.DEBUG): evil_logger.debug("reading full response.") return self._actual.read(amt) def readlines(self): return self._actual.readlines() def readline(self, size=-1): return self._actual.readline(size) return Urllib3LikeResponse(response) def disconnect(self): """Disconnect the current HTTP connection if one exists.""" connection = self._get_connection() if connection is not None: connection.close() def has(self, relpath): """Does the target location exist?""" response = self._head(relpath) code = response.status return code == 200 def get(self, relpath): """Get the file at the given relative path. :param relpath: The relative path to the file """ _code, response_file = self._get(relpath, None) return response_file def _get(self, relpath, offsets, tail_amount=0): """Get a file, or part of a file. :param relpath: Path relative to transport base URL :param offsets: None to get the whole file; or a list of _CoalescedOffset to fetch parts of a file. :param tail_amount: The amount to get from the end of the file. :returns: (http_code, result_file) """ abspath = self._remote_path(relpath) headers = {} if offsets or tail_amount: range_header = self._attempted_range_header(offsets, tail_amount) if range_header is not None: bytes = "bytes=" + range_header headers = {"Range": bytes} else: range_header = None response = self.request("GET", abspath, headers=headers) if response.status == 404: # not found raise NoSuchFile(abspath) elif response.status == 416: # We don't know which, but one of the ranges we specified was # wrong. raise InvalidHttpRange( abspath, range_header, "Server return code %d" % response.status ) elif response.status == 400: if range_header: # We don't know which, but one of the ranges we specified was # wrong. raise InvalidHttpRange( abspath, range_header, "Server return code %d" % response.status ) else: raise BadHttpRequest(abspath, response.reason) elif response.status not in (200, 206): raise UnexpectedHttpStatus( abspath, response.status, headers=response.getheaders() ) data = handle_response(abspath, response.status, response.getheader, response) return response.status, data def _remote_path(self, relpath): """See ConnectedTransport._remote_path. user and passwords are not embedded in the path provided to the server. """ url = self._parsed_url.clone(relpath) url.user = url.quoted_user = None url.password = url.quoted_password = None url.scheme = self._unqualified_scheme return str(url) def _create_auth(self): """Returns a dict containing the credentials provided at build time.""" auth = { "host": self._parsed_url.host, "port": self._parsed_url.port, "user": self._parsed_url.user, "password": self._parsed_url.password, "protocol": self._unqualified_scheme, "path": self._parsed_url.path, } return auth def _degrade_range_hint(self, relpath, ranges): if self._range_hint == "multi": self._range_hint = "single" logger.debug('Retry "%s" with single range request', relpath) elif self._range_hint == "single": self._range_hint = None logger.debug('Retry "%s" without ranges', relpath) else: # We tried all the tricks, but nothing worked, caller must reraise. return False return True # _coalesce_offsets is a helper for readv, it try to combine ranges without # degrading readv performances. _bytes_to_read_before_seek is the value # used for the limit parameter and has been tuned for other transports. For # HTTP, the name is inappropriate but the parameter is still useful and # helps reduce the number of chunks in the response. The overhead for a # chunk (headers, length, footer around the data itself is variable but # around 50 bytes. We use 128 to reduce the range specifiers that appear in # the header, some servers (notably Apache) enforce a maximum length for a # header and issue a '400: Bad request' error when too much ranges are # specified. _bytes_to_read_before_seek = 128 # No limit on the offset number that get combined into one, we are trying # to avoid downloading the whole file. _max_readv_combine = 0 # By default Apache has a limit of ~400 ranges before replying with a 400 # Bad Request. So we go underneath that amount to be safe. _max_get_ranges = 200 # We impose no limit on the range size. But see _pycurl.py for a different # use. _get_max_size = 0 def _readv(self, relpath, offsets): """Get parts of the file at the given relative path. :param offsets: A list of (offset, size) tuples. :param return: A list or generator of (offset, data) tuples """ # offsets may be a generator, we will iterate it several times, so # build a list offsets = list(offsets) try_again = True retried_offset = None while try_again: try_again = False # Coalesce the offsets to minimize the GET requests issued sorted_offsets = sorted(offsets) coalesced = self._coalesce_offsets( sorted_offsets, limit=self._max_readv_combine, fudge_factor=self._bytes_to_read_before_seek, max_size=self._get_max_size, ) # Turn it into a list, we will iterate it several times coalesced = list(coalesced) if debug_logger.isEnabledFor(logging.DEBUG): logger.debug( "http readv of %s offsets => %s collapsed %s", relpath, len(offsets), len(coalesced), ) # Cache the data read, but only until it's been used data_map = {} # We will iterate on the data received from the GET requests and # serve the corresponding offsets respecting the initial order. We # need an offset iterator for that. iter_offsets = iter(offsets) try: cur_offset_and_size = next(iter_offsets) except StopIteration: return try: for cur_coal, rfile in self._coalesce_readv(relpath, coalesced): # Split the received chunk for offset, size in cur_coal.ranges: start = cur_coal.start + offset rfile.seek(start, os.SEEK_SET) data = rfile.read(size) data_len = len(data) if data_len != size: raise ShortReadvError(relpath, start, size, actual=data_len) if (start, size) == cur_offset_and_size: # The offset requested are sorted as the coalesced # ones, no need to cache. Win ! yield cur_offset_and_size[0], data try: cur_offset_and_size = next(iter_offsets) except StopIteration: return else: # Different sorting. We need to cache. data_map[(start, size)] = data # Yield everything we can while cur_offset_and_size in data_map: # Clean the cached data since we use it # XXX: will break if offsets contains duplicates -- # vila20071129 this_data = data_map.pop(cur_offset_and_size) yield cur_offset_and_size[0], this_data try: cur_offset_and_size = next(iter_offsets) except StopIteration: return except ( ShortReadvError, InvalidRange, InvalidHttpRange, HttpBoundaryMissing, ) as e: logger.debug("Exception %r: %s during http._readv", e, e) if ( not isinstance(e, ShortReadvError) or retried_offset == cur_offset_and_size ): # We don't degrade the range hint for ShortReadvError since # they do not indicate a problem with the server ability to # handle ranges. Except when we fail to get back a required # offset twice in a row. In that case, falling back to # single range or whole file should help. if not self._degrade_range_hint(relpath, coalesced): raise # Some offsets may have been already processed, so we retry # only the unsuccessful ones. offsets = [cur_offset_and_size] + list(iter_offsets) retried_offset = cur_offset_and_size try_again = True def _coalesce_readv(self, relpath, coalesced): """Issue several GET requests to satisfy the coalesced offsets.""" def get_and_yield(relpath, coalesced): if coalesced: # Note that the _get below may raise # errors.InvalidHttpRange. It's the caller's responsibility to # decide how to retry since it may provide different coalesced # offsets. _code, rfile = self._get(relpath, coalesced) for coal in coalesced: yield coal, rfile if self._range_hint is None: # Download whole file yield from get_and_yield(relpath, coalesced) else: total = len(coalesced) if self._range_hint == "multi": max_ranges = self._max_get_ranges elif self._range_hint == "single": max_ranges = total else: raise AssertionError(f"Unknown _range_hint {self._range_hint!r}") # TODO: Some web servers may ignore the range requests and return # the whole file, we may want to detect that and avoid further # requests. # Hint: test_readv_multiple_get_requests will fail once we do that cumul = 0 ranges = [] for coal in coalesced: if ( self._get_max_size > 0 and cumul + coal.length > self._get_max_size ) or len(ranges) >= max_ranges: # Get that much and yield yield from get_and_yield(relpath, ranges) # Restart with the current offset ranges = [coal] cumul = coal.length else: ranges.append(coal) cumul += coal.length # Get the rest and yield yield from get_and_yield(relpath, ranges) def recommended_page_size(self): """See Transport.recommended_page_size(). For HTTP we suggest a large page size to reduce the overhead introduced by latency. """ return 64 * 1024 def _post(self, body_bytes): """POST body_bytes to .bzr/smart on this transport. :returns: (response code, response body file-like object). """ # TODO: Requiring all the body_bytes to be available at the beginning of # the POST may require large client buffers. It would be nice to have # an interface that allows streaming via POST when possible (and # degrades to a local buffer when not). abspath = self._remote_path(".bzr/smart") response = self.request( "POST", abspath, body=body_bytes, headers={"Content-Type": "application/octet-stream"}, ) code = response.status data = handle_response(abspath, code, response.getheader, response) return code, data def _head(self, relpath): """Request the HEAD of a file. Performs the request and leaves callers handle the results. """ abspath = self._remote_path(relpath) response = self.request("HEAD", abspath) if response.status not in (200, 404): raise UnexpectedHttpStatus( abspath, response.status, headers=response.getheaders() ) return response raise NotImplementedError(self._post) def put_file(self, relpath, f, mode=None): """Copy the file-like object into the location. :param relpath: Location to put the contents, relative to base. :param f: File-like object. """ raise TransportNotPossible("http PUT not supported") def mkdir(self, relpath, mode=None): """Create a directory at the given path.""" raise TransportNotPossible("http does not support mkdir()") def rmdir(self, relpath): """See Transport.rmdir.""" raise TransportNotPossible("http does not support rmdir()") def append_file(self, relpath, f, mode=None): """Append the text in the file-like object into the final location. """ raise TransportNotPossible("http does not support append()") def copy(self, rel_from, rel_to): """Copy the item at rel_from to the location at rel_to.""" raise TransportNotPossible("http does not support copy()") def copy_to(self, relpaths, other, mode=None, pb=None): """Copy a set of entries from self into another Transport. :param relpaths: A list/generator of entries to be copied. TODO: if other is LocalTransport, is it possible to do better than put(get())? """ # At this point HttpTransport might be able to check and see if # the remote location is the same, and rather than download, and # then upload, it could just issue a remote copy_this command. if isinstance(other, HttpTransport): raise TransportNotPossible("http cannot be the target of copy_to()") else: return super().copy_to(relpaths, other, mode=mode, pb=pb) def move(self, rel_from, rel_to): """Move the item at rel_from to the location at rel_to.""" raise TransportNotPossible("http does not support move()") def delete(self, relpath): """Delete the item at relpath.""" raise TransportNotPossible("http does not support delete()") def external_url(self): """See dromedary.Transport.external_url.""" # HTTP URL's are externally usable as long as they don't mention their # implementation qualifier url = self._parsed_url.clone() url.scheme = self._unqualified_scheme return str(url) def is_readonly(self): """See Transport.is_readonly.""" return True def listable(self): """See Transport.listable.""" return False def stat(self, relpath): """Return the stat information for a file.""" raise TransportNotPossible("http does not support stat()") def lock_read(self, relpath): """Lock the given file for shared (read) access. :return: A lock object, which should be passed to Transport.unlock(). """ # The old RemoteBranch ignore lock for reading, so we will # continue that tradition and return a bogus lock object. class BogusLock: def __init__(self, path): self.path = path def unlock(self): pass return BogusLock(relpath) def lock_write(self, relpath): """Lock the given file for exclusive (write) access. WARNING: many transports do not support this, so trying avoid using it. :return: A lock object, which should be passed to Transport.unlock() """ raise TransportNotPossible("http does not support lock_write()") def _attempted_range_header(self, offsets, tail_amount): """Prepare a HTTP Range header at a level the server should accept. :return: the range header representing offsets/tail_amount or None if no header can be built. """ if self._range_hint == "multi": # Generate the header describing all offsets return self._range_header(offsets, tail_amount) elif self._range_hint == "single": # Combine all the requested ranges into a single # encompassing one if len(offsets) > 0: if tail_amount not in (0, None): # Nothing we can do here to combine ranges with tail_amount # in a single range, just returns None. The whole file # should be downloaded. return None else: start = offsets[0].start last = offsets[-1] end = last.start + last.length - 1 whole = self._coalesce_offsets( [(start, end - start + 1)], limit=0, fudge_factor=0 ) return self._range_header(list(whole), 0) else: # Only tail_amount, requested, leave range_header # do its work return self._range_header(offsets, tail_amount) else: return None @staticmethod def _range_header(ranges, tail_amount): """Turn a list of bytes ranges into a HTTP Range header value. :param ranges: A list of _CoalescedOffset :param tail_amount: The amount to get from the end of the file. :return: HTTP range header string. At least a non-empty ranges *or* a tail_amount must be provided. """ strings = [] for offset in ranges: strings.append("%d-%d" % (offset.start, offset.start + offset.length - 1)) if tail_amount: strings.append("-%d" % tail_amount) return ",".join(strings) def _redirected_to(self, source, target): """Returns a transport suitable to re-issue a redirected request. :param source: The source url as returned by the server. :param target: The target url as returned by the server. The redirection can be handled only if the relpath involved is not renamed by the redirection. :returns: A transport :raise UnusableRedirect: when the URL can not be reinterpreted """ parsed_source = self._split_url(source) parsed_target = self._split_url(target) pl = len(self._parsed_url.path) # determine the excess tail - the relative path that was in # the original request but not part of this transports' URL. excess_tail = parsed_source.path[pl:].strip("/") if not parsed_target.path.endswith(excess_tail): # The final part of the url has been renamed, we can't handle the # redirection. raise UnusableRedirect(source, target, "final part of the url was renamed") target_path = parsed_target.path if excess_tail: # Drop the tail that was in the redirect but not part of # the path of this transport. target_path = target_path[: -len(excess_tail)] if parsed_target.scheme in ("http", "https"): # Same protocol family (i.e. http[s]), we will preserve the same # http client implementation when a redirection occurs from one to # the other (otherwise users may be surprised that bzr switches # from one implementation to the other, and devs may suffer # debugging it). if ( parsed_target.scheme == self._unqualified_scheme and parsed_target.host == self._parsed_url.host and parsed_target.port == self._parsed_url.port and ( parsed_target.user is None or parsed_target.user == self._parsed_url.user ) ): # If a user is specified, it should match, we don't care about # passwords, wrong passwords will be rejected anyway. return self.clone(target_path) else: # Rebuild the url preserving the scheme qualification and the # credentials (if they don't apply, the redirected to server # will tell us, but if they do apply, we avoid prompting the # user) redir_scheme = parsed_target.scheme new_url = self._unsplit_url( redir_scheme, self._parsed_url.user, self._parsed_url.password, parsed_target.host, parsed_target.port, target_path, ) return _mod_dromedary.get_transport_from_url(new_url) else: # Redirected to a different protocol new_url = self._unsplit_url( parsed_target.scheme, parsed_target.user, parsed_target.password, parsed_target.host, parsed_target.port, target_path, ) return _mod_dromedary.get_transport_from_url(new_url) def _options(self, relpath): abspath = self._remote_path(relpath) resp = self.request("OPTIONS", abspath) if resp.status == 404: raise NoSuchFile(abspath) if resp.status in (403, 405): raise InvalidHttpResponse( abspath, "OPTIONS not supported or forbidden for remote URL", headers=resp.getheaders(), ) return resp.getheaders() def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import http_server permutations = [ (HttpTransport, http_server.HttpServer), ] try: import ssl # noqa: F401 from dromedary.tests import https_server, ssl_certs class HTTPS_transport(HttpTransport): def __init__(self, base, _from_transport=None): super().__init__( base, _from_transport=_from_transport, ca_certs=ssl_certs.build_path("ca.crt"), ) permutations.append((HTTPS_transport, https_server.HTTPSServer)) except ModuleNotFoundError: pass return permutations dromedary-0.1.1/local.py000064400000000000000000000070351046102023000132620ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport for the local filesystem. This is a fairly thin wrapper on regular file IO. """ import os from dromedary import errors, urlutils from dromedary.osutils import _win32_normpath, file_kind_from_stat_mode, pathjoin def file_stat(f, _lstat=os.lstat): """Get stat information for a file. Args: f: Path to the file. _lstat: Function to use for stat (defaults to os.lstat). Returns: Stat result object. Raises: NoSuchFile: If the file doesn't exist. """ try: return _lstat(f) except (FileNotFoundError, NotADirectoryError) as err: raise errors.NoSuchFile(f) from err def file_kind(f, _lstat=os.lstat): """Determine the kind of file (regular, directory, symlink, etc). Args: f: Path to the file. _lstat: Function to use for stat (defaults to os.lstat). Returns: String describing the file kind ('file', 'directory', 'symlink', etc). """ stat_value = file_stat(f, _lstat) return file_kind_from_stat_mode(stat_value.st_mode) from ._transport_rs.local import LocalTransport # type:ignore class EmulatedWin32LocalTransport(LocalTransport): # type:ignore """Special transport for testing Win32 [UNC] paths on non-windows.""" def __init__(self, base): """Initialize EmulatedWin32LocalTransport. Args: base: Base URL for the transport. """ if base[-1] != "/": base = base + "/" super(LocalTransport, self).__init__(base) self._local_base = urlutils._win32_local_path_from_url(base) def abspath(self, relpath): """Return the absolute URL for a relative path. Args: relpath: Relative path from the transport base. Returns: Absolute URL using Win32 path conventions. """ path = _win32_normpath(pathjoin(self._local_base, urlutils.unescape(relpath))) return urlutils._win32_local_path_to_url(path) def clone(self, offset=None): """Return a new LocalTransport with root at self.base + offset Because the local filesystem does not require a connection, we can just return a new object. """ if offset is None: return EmulatedWin32LocalTransport(self.base) else: abspath = self.abspath(offset) if abspath == "file://": # fix upwalk for UNC path # when clone from //HOST/path updir recursively # we should stop at least at //HOST part abspath = self.base return EmulatedWin32LocalTransport(abspath) def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [ (LocalTransport, test_server.LocalURLServer), ] dromedary-0.1.1/log.py000064400000000000000000000147311046102023000127520ustar 00000000000000# Copyright (C) 2008, 2009, 2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport decorator that logs transport operations.""" # see also the transportstats plugin, which gives you some summary information # in a machine-readable dump import logging import time import types from dromedary import decorator logger = logging.getLogger("dromedary.log") class TransportLogDecorator(decorator.TransportDecorator): """Decorator for Transports that logs interesting operations to brz.log. In general we want to log things that usually take a network round trip and may be slow. Not all operations are logged yet. See also TransportTraceDecorator, that records a machine-readable log in memory for eg testing. """ def __init__(self, *args, **kw): """Initialize the TransportLogDecorator. Args: *args: Arguments passed to parent TransportDecorator. **kw: Keyword arguments passed to parent TransportDecorator. """ super().__init__(*args, **kw) def _make_hook(hookname): def _hook(relpath, *args, **kw): return self._log_and_call(hookname, relpath, *args, **kw) return _hook # GZ 2017-05-21: Not all methods take relpath as first argument, for # instance copy_to takes list of relpaths. Also, unclear on url vs # filesystem path split. Needs tidying up. for methodname in ( "append_bytes", "append_file", "copy_to", "delete", "get", "has", "open_write_stream", "mkdir", "move", "put_bytes", "put_bytes_non_atomic", "put_file", "put_file_non_atomic", "list_dir", "lock_read", "lock_write", "readv", "rename", "rmdir", "stat", "ulock", ): setattr(self, methodname, _make_hook(methodname)) @classmethod def _get_url_prefix(self): return "log+" def iter_files_recursive(self): """Iterate through all files recursively. Returns: Iterator of file paths beneath this transport. """ # needs special handling because it does not have a relpath parameter logger.debug("iter_files_recursive %s", self._decorated.base) return self._call_and_log_result("iter_files_recursive", (), {}) def _log_and_call(self, methodname, relpath, *args, **kwargs): kwargs_str = dict(kwargs) if kwargs else "" logger.debug( "%s %s %s %s", methodname, relpath, self._shorten(self._strip_tuple_parens(args)), kwargs_str, ) return self._call_and_log_result(methodname, (relpath,) + args, kwargs) def _call_and_log_result(self, methodname, args, kwargs): before = time.time() try: result = getattr(self._decorated, methodname)(*args, **kwargs) except Exception as e: logger.debug(" --> %s", e) logger.debug(" %.03fs", time.time() - before) raise return self._show_result(before, methodname, result) def _show_result(self, before, methodname, result): result_len = None if ( isinstance(result, types.GeneratorType) or type(result).__name__ == "list_iterator" ): # We now consume everything from the generator so that we can show # the results and the time it took to get them. However, to keep # compatibility with callers that may specifically expect a result # (see ) we also return a new # generator, reset to the starting position. result = list(result) return_result = iter(result) else: return_result = result # Is this an io object with a getvalue() method? getvalue = getattr(result, "getvalue", None) if getvalue is not None: val = repr(getvalue()) result_len = len(val) shown_result = "%s(%s) (%d bytes)" % ( result.__class__.__name__, self._shorten(val), result_len, ) elif methodname == "readv": num_hunks = len(result) total_bytes = sum((len(d) for o, d in result)) shown_result = "readv response, %d hunks, %d total bytes" % ( num_hunks, total_bytes, ) result_len = total_bytes else: shown_result = self._shorten(self._strip_tuple_parens(result)) logger.debug(" --> %s", shown_result) # The log decorator no longer shows the elapsed time or transfer rate # because they're available in the log prefixes and the transport # activity display respectively. if False: elapsed = time.time() - before if result_len and elapsed > 0: # this is the rate of higher-level data, not the raw network # speed using base-10 units (see HACKING.txt). logger.debug( " %9.03fs %8dkB/s", elapsed, result_len / elapsed / 1000 ) else: logger.debug(" %9.03fs", elapsed) return return_result def _shorten(self, x): if len(x) > 70: x = x[:67] + "..." return x def _strip_tuple_parens(self, t): t = repr(t) if t[0] == "(" and t[-1] == ")": t = t[1:-1] return t def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(TransportLogDecorator, test_server.LogDecoratorServer)] dromedary-0.1.1/memory.py000064400000000000000000000046541046102023000135040ustar 00000000000000# Copyright (C) 2005-2011, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport that uses memory for its storage. The contents of the transport will be lost when the object is discarded, so this is primarily useful for testing. """ from dromedary import Server, register_transport, unregister_transport from dromedary._transport_rs.memory import MemoryStoreHandle from dromedary._transport_rs.memory import MemoryTransport as _RustMemoryTransport __all__ = ["MemoryServer", "MemoryTransport", "get_test_permutations"] class MemoryTransport(_RustMemoryTransport): """This is an in memory file system for transient data storage.""" class MemoryServer(Server): """Server for the MemoryTransport for testing with.""" def start_server(self): """Start the memory server by initializing storage and registering transport.""" self._store = MemoryStoreHandle() self._scheme = f"memory+{id(self)}:///" def memory_factory(url): return MemoryTransport(url, _shared_store=self._store) self._memory_factory = memory_factory register_transport(self._scheme, self._memory_factory) def stop_server(self): """Stop the server and unregister the transport.""" unregister_transport(self._scheme, self._memory_factory) def get_url(self): """See dromedary.Server.get_url.""" return self._scheme def get_bogus_url(self): """Get a URL for a non-existent location. Raises: NotImplementedError: This method is not implemented for memory transport. """ raise NotImplementedError def get_test_permutations(): """Return the permutations to be used in testing.""" return [ (MemoryTransport, MemoryServer), ] dromedary-0.1.1/osutils.py000064400000000000000000000213321046102023000136660ustar 00000000000000#!/usr/bin/env python3 # Copyright (C) 2005-2024 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Operating system utilities for dromedary transport layer.""" import errno import fcntl import os import random import stat import string import sys def pumpfile(from_file, to_file, read_length=-1, buff_size=32768): """Copy bytes from from_file to to_file, optionally limiting total length. Args: from_file: File-like object to read from. to_file: File-like object to write to. read_length: Total number of bytes to copy. -1 (the default) means copy to EOF. buff_size: Size of each individual read. Returns: The total number of bytes copied. """ written = 0 if read_length is not None and read_length >= 0: # Read exactly read_length bytes total, in buff_size chunks. bytes_left = read_length while bytes_left > 0: chunk = from_file.read(min(buff_size, bytes_left)) if not chunk: break to_file.write(chunk) bytes_left -= len(chunk) written += len(chunk) else: while True: chunk = from_file.read(buff_size) if not chunk: break to_file.write(chunk) written += len(chunk) return written def pump_string_file(string, to_file, segment_size=8192): """Write a string to a file efficiently. Args: string: String or bytes to write to_file: File-like object to write to segment_size: Size of chunks to write """ if isinstance(string, str): string = string.encode("utf-8") offset = 0 while offset < len(string): segment = string[offset : offset + segment_size] to_file.write(segment) offset += len(segment) def fancy_rename(old, new, rename_func, unlink_func): """A fancy rename, when you don't have atomic rename. :param old: The old path, to rename from :param new: The new path, to rename to :param rename_func: The potentially non-atomic rename function :param unlink_func: A way to delete the target file if the full rename succeeds """ import time from dromedary.errors import NoSuchFile # sftp rename doesn't allow overwriting, so play tricks: base = os.path.basename(new) dirname = os.path.dirname(new) tmp_name = "tmp.%s.%.9f.%d.%s" % (base, time.time(), os.getpid(), rand_chars(10)) tmp_name = pathjoin(dirname, tmp_name) # Rename the file out of the way, but keep track if it didn't exist file_existed = False try: rename_func(new, tmp_name) except NoSuchFile: pass except (FileNotFoundError, NotADirectoryError): pass except OSError: # paramiko SFTP rename raises IOError with errno=None on failure. raise except Exception as e: if getattr(e, "errno", None) is None or e.errno not in ( errno.ENOENT, errno.ENOTDIR, ): raise else: file_existed = True success = False try: rename_func(old, new) success = True except FileNotFoundError: # source and target may be aliases of each other on a # case-insensitive filesystem if file_existed and old.lower() == new.lower(): pass else: raise finally: if file_existed: if success: unlink_func(tmp_name) else: rename_func(tmp_name, new) def fdatasync(fileno): """Force data to be written to disk. Args: fileno: File descriptor or file object with fileno() method """ if hasattr(fileno, "fileno"): fileno = fileno.fileno() if hasattr(os, "fdatasync"): os.fdatasync(fileno) elif hasattr(os, "fsync"): os.fsync(fileno) # If neither is available, do nothing (some platforms don't support this) def file_kind_from_stat_mode(stat_mode): """Determine file type from stat mode bits. Args: stat_mode: Mode from os.stat() Returns: String describing file type: 'file', 'directory', 'symlink' """ if stat.S_ISREG(stat_mode): return "file" elif stat.S_ISDIR(stat_mode): return "directory" elif stat.S_ISLNK(stat_mode): return "symlink" elif stat.S_ISCHR(stat_mode): return "chardev" elif stat.S_ISBLK(stat_mode): return "block" elif stat.S_ISFIFO(stat_mode): return "fifo" elif stat.S_ISSOCK(stat_mode): return "socket" else: return "unknown" def set_fd_cloexec(fd): """Set the close-on-exec flag for a file descriptor. Args: fd: File descriptor or file object with fileno() method """ if hasattr(fd, "fileno"): fd = fd.fileno() if hasattr(fcntl, "FD_CLOEXEC"): flags = fcntl.fcntl(fd, fcntl.F_GETFD) fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) def rand_chars(n): """Generate random characters. Args: n: Number of characters to generate Returns: String of random alphanumeric characters """ chars = string.ascii_letters + string.digits return "".join(random.choice(chars) for _ in range(n)) # noqa: S311 def splitpath(path): """Split a path into components. Args: path: Path to split Returns: List of path components """ if not path or path == "/": return [] # Remove leading slash for consistent behavior if path.startswith("/"): path = path[1:] # Remove trailing slash if path.endswith("/"): path = path[:-1] if not path: return [] return path.split("/") def pathjoin(*args): """Join path components, handling various edge cases. Args: *args: Path components to join Returns: Joined path """ if not args: return "" # Filter out empty components components = [arg for arg in args if arg and arg != "."] if not components: return "" # Join with forward slashes (transport paths use forward slashes) result = "/".join(components) # Handle absolute paths if args[0].startswith("/"): result = "/" + result return result def get_terminal_encoding(): """Get the terminal's character encoding. Returns: String name of encoding, defaults to 'utf-8' """ import locale # Try to get the terminal encoding encoding = None if hasattr(sys.stdout, "encoding") and sys.stdout.encoding: encoding = sys.stdout.encoding if not encoding: try: encoding = locale.getpreferredencoding() except Exception: pass if not encoding: encoding = "utf-8" # Safe default return encoding def getcwd(): """Return the current working directory as a unicode string.""" return os.getcwd() def abspath(path): """Return the absolute version of a path.""" return os.path.abspath(path) def get_umask(): """Return the current umask.""" umask = os.umask(0) os.umask(umask) return umask def supports_symlinks(path=None): """Return True if the filesystem supports symlinks.""" return getattr(os, "symlink", None) is not None def get_user_encoding(): """Return the encoding used for user-facing text.""" return get_terminal_encoding() def _posix_normpath(path): return os.path.normpath(path) normpath = os.path.normpath split = os.path.split MIN_ABS_PATHLENGTH = 3 if sys.platform == "win32" else 1 def _win32_abspath(path): return os.path.abspath(path) def _win32_normpath(path): """Normalize a Windows path. This is used on Windows to normalize path separators and handle drive letters properly. Args: path: Path to normalize Returns: Normalized path string """ if sys.platform == "win32": # On Windows, normalize path separators and handle drive letters import os.path return os.path.normpath(path).replace("\\", "/") else: # On non-Windows, just return the path as-is return path dromedary-0.1.1/pathfilter.py000064400000000000000000000045771046102023000143420ustar 00000000000000# Copyright (C) 2009, 2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """A transport decorator that filters all paths that are passed to it.""" from dromedary import Server, register_transport, unregister_transport from dromedary._transport_rs.pathfilter import PathFilteringTransport __all__ = [ "PathFilteringServer", "PathFilteringTransport", "get_test_permutations", ] class PathFilteringServer(Server): """Transport server for PathFilteringTransport. It holds the backing_transport and filter_func for PathFilteringTransports. All paths will be passed through filter_func before calling into the backing_transport. """ def __init__(self, backing_transport, filter_func): """Constructor. :param backing_transport: a transport :param filter_func: a callable that takes paths, and translates them into paths for use with the backing transport. """ self.backing_transport = backing_transport self.filter_func = filter_func def _factory(self, url): return PathFilteringTransport(self, url) def get_url(self): """Return the URL scheme for this server.""" return self.scheme def start_server(self): """Start the path filtering transport server.""" self.scheme = "filtered-%d:///" % id(self) register_transport(self.scheme, self._factory) def stop_server(self): """Stop the path filtering transport server.""" unregister_transport(self.scheme, self._factory) def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(PathFilteringTransport, test_server.TestingPathFilteringServer)] dromedary-0.1.1/py.typed000064400000000000000000000000001046102023000132760ustar 00000000000000dromedary-0.1.1/readonly.py000064400000000000000000000022521046102023000140010ustar 00000000000000# Copyright (C) 2006, 2007, 2009, 2010, 2011, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport that adapts another transport to be readonly.""" from dromedary._transport_rs.readonly import ReadonlyTransportDecorator __all__ = ["ReadonlyTransportDecorator", "get_test_permutations"] def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(ReadonlyTransportDecorator, test_server.ReadonlyServer)] dromedary-0.1.1/setup.py000064400000000000000000000035161046102023000133300ustar 00000000000000#!/usr/bin/env python3 """Installation script for dromedary. Dromedary is the transport layer abstraction extracted from Breezy. """ import os # Import version from version module import sys from setuptools import find_packages, setup sys.path.insert(0, os.path.dirname(__file__)) from version import version_string try: from setuptools_rust import Binding, RustExtension except ModuleNotFoundError: RustExtension = None rust_extensions = [] else: rust_extensions = [ RustExtension( "dromedary._transport_rs", "_transport_rs/Cargo.toml", binding=Binding.PyO3 ), ] with open("README.md", encoding="utf-8") as fh: long_description = fh.read() setup( name="dromedary", version=version_string, author="Breezy Team", author_email="team@breezy-vcs.org", description="Transport layer abstraction for version control systems", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/breezy-team/dromedary", packages=find_packages(), rust_extensions=rust_extensions, classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Version Control", ], python_requires=">=3.8", install_requires=[], extras_require={ "sftp": ["paramiko"], "gio": ["pygobject"], }, zip_safe=False, ) dromedary-0.1.1/sftp.py000064400000000000000000001123031046102023000131370ustar 00000000000000# Copyright (C) 2005-2011, 2016, 2017 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport over SFTP, using paramiko.""" # TODO: Remove the transport-based lock_read and lock_write methods. They'll # then raise TransportNotPossible, which will break remote access to any # formats which rely on OS-level locks. That should be fine as those formats # are pretty old, but these combinations may have to be removed from the test # suite. Those formats all date back to 0.7; so we should be able to remove # these methods when we officially drop support for those formats. import bisect import errno import itertools import logging import os import random import stat import sys import time from dromedary import ( ConnectedTransport, FileFileStream, _config, _file_streams, errors, ssh, urlutils, ) from dromedary.errors import ( DependencyNotPresent, FileExists, LockContention, NoSuchFile, PathError, ReadError, TransportNotPossible, ) from dromedary.osutils import fancy_rename, pumpfile logger = logging.getLogger("dromedary.sftp") debug_logger = logging.getLogger("dromedary.sftp") from ._transport_rs import sftp as _sftp_rs SFTPError = _sftp_rs.SFTPError class ParamikoNotPresent(DependencyNotPresent): """Paramiko library is not available. Raised when paramiko is required for SFTP support but is not installed or cannot be imported. """ _fmt = "Unable to import paramiko (required for sftp support): %(error)s" def __init__(self, error): """Initialize with paramiko import error. Args: error: The import error that occurred when trying to import paramiko. """ DependencyNotPresent.__init__(self, "paramiko", error) class WriteStream: """Simple write stream wrapper for SFTP file objects.""" def __init__(self, f): """Initialize write stream. Args: f: SFTP file object to wrap. """ self.f = f def write(self, data): """Write data to the stream. Args: data: Bytes to write. Returns: int: Number of bytes written. """ self.f.write(data) return len(data) class SFTPLock: """This fakes a lock in a remote location. A present lock is indicated just by the existence of a file. This doesn't work well on all transports and they are only used in deprecated storage formats. """ __slots__ = ["lock_file", "lock_path", "path", "transport"] def __init__(self, path, transport): """Initialize SFTP lock. Args: path: Path to lock. transport: SFTP transport to use. Raises: LockContention: If the file is already locked. """ self.lock_file = None self.path = path self.lock_path = path + ".write-lock" self.transport = transport try: # RBC 20060103 FIXME should we be using private methods here ? abspath = transport._remote_path(self.lock_path) self.lock_file = transport._sftp_open_exclusive(abspath) except FileExists as err: raise LockContention(self.path) from err def unlock(self): """Release the lock by closing and deleting the lock file.""" if not self.lock_file: return self.lock_file.close() self.lock_file = None try: self.transport.delete(self.lock_path) except NoSuchFile: # What specific errors should we catch here? pass class _SFTPReadvHelper: """A class to help with managing the state of a readv request.""" def __init__(self, original_offsets, relpath, _report_activity): """Create a new readv helper. :param original_offsets: The original requests given by the caller of readv() :param relpath: The name of the file (if known) :param _report_activity: A Transport._report_activity bound method, to be called as data arrives. """ self.original_offsets = list(original_offsets) self.relpath = relpath self._report_activity = _report_activity def _get_requests(self): """Break up the offsets into individual requests over sftp. The SFTP spec only requires implementers to support 32kB requests. We could try something larger (openssh supports 64kB), but then we have to handle requests that fail. So instead, we just break up our maximum chunks into 32kB chunks, and asyncronously requests them. Newer versions of paramiko would do the chunking for us, but we want to start processing results right away, so we do it ourselves. """ # TODO: Because we issue async requests, we don't 'fudge' any extra # data. I'm not 100% sure that is the best choice. # The first thing we do, is to collapse the individual requests as much # as possible, so we don't issues requests <32kB sorted_offsets = sorted(self.original_offsets) coalesced = list( ConnectedTransport._coalesce_offsets( sorted_offsets, limit=0, fudge_factor=0 ) ) requests = [(c_offset.start, c_offset.length) for c_offset in coalesced] debug_logger.debug( "SFTP.readv(%s) %s offsets => %s coalesced => %s requests", self.relpath, len(sorted_offsets), len(coalesced), len(requests), ) return requests def request_and_yield_offsets(self, fp): """Request the data from the remote machine, yielding the results. :param fp: A Paramiko SFTPFile object that supports readv. :return: Yield the data requested by the original readv caller, one by one. """ requests = self._get_requests() offset_iter = iter(self.original_offsets) cur_offset, cur_size = next(offset_iter) # paramiko .readv() yields strings that are in the order of the requests # So we track the current request to know where the next data is # being returned from. input_start = None last_end = None buffered_data = [] buffered_len = 0 # This is used to buffer chunks which we couldn't process yet # It is (start, end, data) tuples. data_chunks = [] # Create an 'unlimited' data stream, so we stop based on requests, # rather than just because the data stream ended. This lets us detect # short readv. data_stream = itertools.chain(fp.readv(requests), itertools.repeat(None)) for (start, length), data in zip(requests, data_stream, strict=False): if data is None and cur_coalesced is not None: raise errors.ShortReadvError(self.relpath, start, length, len(data)) if len(data) != length: raise errors.ShortReadvError(self.relpath, start, length, len(data)) self._report_activity(length, "read") if last_end is None: # This is the first request, just buffer it buffered_data = [data] buffered_len = length input_start = start elif start == last_end: # The data we are reading fits neatly on the previous # buffer, so this is all part of a larger coalesced range. buffered_data.append(data) buffered_len += length else: # We have an 'interrupt' in the data stream. So we know we are # at a request boundary. if buffered_len > 0: # We haven't consumed the buffer so far, so put it into # data_chunks, and continue. buffered = b"".join(buffered_data) data_chunks.append((input_start, buffered)) input_start = start buffered_data = [data] buffered_len = length last_end = start + length if input_start == cur_offset and cur_size <= buffered_len: # Simplify the next steps a bit by transforming buffered_data # into a single string. We also have the nice property that # when there is only one string ''.join([x]) == x, so there is # no data copying. buffered = b"".join(buffered_data) # Clean out buffered data so that we keep memory # consumption low del buffered_data[:] buffered_offset = 0 # TODO: We *could* also consider the case where cur_offset is in # in the buffered range, even though it doesn't *start* # the buffered range. But for packs we pretty much always # read in order, so you won't get any extra data in the # middle. while ( input_start == cur_offset and (buffered_offset + cur_size) <= buffered_len ): # We've buffered enough data to process this request, spit it # out cur_data = buffered[buffered_offset : buffered_offset + cur_size] # move the direct pointer into our buffered data buffered_offset += cur_size # Move the start-of-buffer pointer input_start += cur_size # Yield the requested data yield cur_offset, cur_data try: cur_offset, cur_size = next(offset_iter) except StopIteration: return # at this point, we've consumed as much of buffered as we can, # so break off the portion that we consumed if buffered_offset == len(buffered_data): # No tail to leave behind buffered_data = [] buffered_len = 0 else: buffered = buffered[buffered_offset:] buffered_data = [buffered] buffered_len = len(buffered) # now that the data stream is done, close the handle fp.close() if buffered_len: buffered = b"".join(buffered_data) del buffered_data[:] data_chunks.append((input_start, buffered)) if data_chunks: debug_logger.debug( "SFTP readv left with %d out-of-order bytes", sum(len(x[1]) for x in data_chunks), ) # We've processed all the readv data, at this point, anything we # couldn't process is in data_chunks. This doesn't happen often, so # this code path isn't optimized # We use an interesting process for data_chunks # Specifically if we have "bisect_left([(start, len, entries)], # (qstart,)]) # If start == qstart, then we get the specific node. Otherwise we # get the previous node while True: idx = bisect.bisect_left(data_chunks, (cur_offset,)) if idx < len(data_chunks) and data_chunks[idx][0] == cur_offset: # The data starts here data = data_chunks[idx][1][:cur_size] elif idx > 0: # The data is in a portion of a previous page idx -= 1 sub_offset = cur_offset - data_chunks[idx][0] data = data_chunks[idx][1] data = data[sub_offset : sub_offset + cur_size] else: # We are missing the page where the data should be found, # something is wrong data = "" if len(data) != cur_size: raise AssertionError( "We must have miscalulated." " We expected %d bytes, but only found %d" % (cur_size, len(data)) ) yield cur_offset, data try: cur_offset, cur_size = next(offset_iter) except StopIteration: return class SFTPTransport(ConnectedTransport): """Transport implementation for SFTP access.""" # TODO: jam 20060717 Conceivably these could be configurable, either # by auto-tuning at run-time, or by a configuration (per host??) # but the performance curve is pretty flat, so just going with # reasonable defaults. _max_readv_combine = 200 # Having to round trip to the server means waiting for a response, # so it is better to download extra bytes. # 8KiB had good performance for both local and remote network operations _bytes_to_read_before_seek = 8192 def _pump(self, infile, outfile): return pumpfile(infile, WriteStream(outfile)) def _remote_path(self, relpath): """Return the path to be passed along the sftp protocol for relpath. :param relpath: is a urlencoded string. """ remote_path = self._parsed_url.clone(relpath).path # the initial slash should be removed from the path, and treated as a # homedir relative path (the path begins with a double slash if it is # absolute). see draft-ietf-secsh-scp-sftp-ssh-uri-03.txt # RBC 20060118 we are not using this as its too user hostile. instead # we are following lftp and using /~/foo to mean '~/foo' # vila--20070602 and leave absolute paths begin with a single slash. if remote_path.startswith("/~/"): remote_path = remote_path[3:] elif remote_path == "/~": remote_path = "" return remote_path def _create_connection(self, credentials=None): """Create a new connection with the provided credentials. :param credentials: The credentials needed to establish the connection. :return: The created connection and its associated credentials. The credentials are only the password as it may have been entered interactively by the user and may be different from the one provided in base url at transport creation time. """ password = self._parsed_url.password if credentials is None else credentials vendor = ssh._get_ssh_vendor() user = self._parsed_url.user if user is None: user = _config.get_auth_user( "ssh", self._parsed_url.host, self._parsed_url.port ) connection = vendor.connect_sftp( self._parsed_url.user, password, self._parsed_url.host, self._parsed_url.port, ) return connection, (user, password) def disconnect(self): """Disconnect the current SFTP connection.""" connection = self._get_connection() if connection is not None: connection.close() def _get_sftp(self): """Ensures that a connection is established.""" connection = self._get_connection() if connection is None: # First connection ever connection, credentials = self._create_connection() self._set_connection(connection, credentials) return connection def has(self, relpath): """Does the target location exist?""" try: self._get_sftp().stat(self._remote_path(relpath)) # stat result is about 20 bytes, let's say self._report_activity(20, "read") return True except NoSuchFile: return False def get(self, relpath): """Get the file at the given relative path. :param relpath: The relative path to the file """ try: path = self._remote_path(relpath) f = self._get_sftp().file(path, mode="rb") size = f.stat().st_size if getattr(f, "prefetch", None) is not None: f.prefetch(size) return f except (OSError, SFTPError) as e: self._translate_io_exception( e, path, ": error retrieving", failure_exc=ReadError ) def get_bytes(self, relpath): """Get the contents of a file as a byte string. Args: relpath: Path to the file relative to transport root. Returns: bytes: The file contents. """ # reimplement this here so that we can report how many bytes came back with self.get(relpath) as f: bytes = f.read() self._report_activity(len(bytes), "read") return bytes def _readv(self, relpath, offsets): """See Transport.readv().""" # We overload the default readv() because we want to use a file # that does not have prefetch enabled. # Also, if we have a new paramiko, it implements an async readv() if not offsets: return try: path = self._remote_path(relpath) fp = self._get_sftp().file(path, mode="rb") readv = getattr(fp, "readv", None) if readv: return self._sftp_readv(fp, offsets, relpath) debug_logger.debug("seek and read %s offsets", len(offsets)) return self._seek_and_read(fp, offsets, relpath) except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": error retrieving") def recommended_page_size(self): """See Transport.recommended_page_size(). For SFTP we suggest a large page size to reduce the overhead introduced by latency. """ return 64 * 1024 def _sftp_readv(self, fp, offsets, relpath): """Use the readv() member of fp to do async readv. Then read them using paramiko.readv(). paramiko.readv() does not support ranges > 64K, so it caps the request size, and just reads until it gets all the stuff it wants. """ helper = _SFTPReadvHelper(offsets, relpath, self._report_activity) return helper.request_and_yield_offsets(fp) def put_file(self, relpath, f, mode=None): """Copy the file-like object into the location. :param relpath: Location to put the contents, relative to base. :param f: File-like object. :param mode: The final mode for the file """ final_path = self._remote_path(relpath) return self._put(final_path, f, mode=mode) def _put(self, abspath, f, mode=None): """Helper function so both put() and copy_abspaths can reuse the code.""" tmp_abspath = "%s.tmp.%.9f.%d.%d" % ( abspath, time.time(), os.getpid(), random.randint(0, 0x7FFFFFFF), # noqa: S311 ) fout = self._sftp_open_exclusive(tmp_abspath, mode=mode) closed = False try: try: length = self._pump(f, fout) except (OSError, SFTPError) as e: self._translate_io_exception(e, tmp_abspath) # XXX: This doesn't truly help like we would like it to. # The problem is that openssh strips sticky bits. So while we # can properly set group write permission, we lose the group # sticky bit. So it is probably best to stop chmodding, and # just tell users that they need to set the umask correctly. # The attr.st_mode = mode, in _sftp_open_exclusive # will handle when the user wants the final mode to be more # restrictive. And then we avoid a round trip. Unless # paramiko decides to expose an async chmod() # This is designed to chmod() right before we close. # Because we set_pipelined() earlier, theoretically we might # avoid the round trip for fout.close() if mode is not None: self._get_sftp().chmod(tmp_abspath, mode) fout.close() closed = True self._rename_and_overwrite(tmp_abspath, abspath) return length except Exception as e: # If we fail, try to clean up the temporary file # before we throw the exception # but don't let another exception mess things up # Write out the traceback, because otherwise # the catch and throw destroys it import traceback logger.debug("%s", traceback.format_exc()) try: if not closed: fout.close() self._get_sftp().remove(tmp_abspath) except BaseException: # raise the saved except raise e from None # raise the original with its traceback if we can. raise def _put_non_atomic_helper( self, relpath, writer, mode=None, create_parent_dir=False, dir_mode=None ): abspath = self._remote_path(relpath) # TODO: jam 20060816 paramiko doesn't publicly expose a way to # set the file mode at create time. If it does, use it. # But for now, we just chmod later anyway. def _open_and_write_file(): """Try to open the target file, raise error on failure.""" fout = None try: try: fout = self._get_sftp().file(abspath, mode="wb") writer(fout) except (SFTPError, OSError) as e: self._translate_io_exception(e, abspath, ": unable to open") # This is designed to chmod() right before we close. # Because we set_pipelined() earlier, theoretically we might # avoid the round trip for fout.close() if mode is not None: self._get_sftp().chmod(abspath, mode) finally: if fout is not None: fout.close() if not create_parent_dir: _open_and_write_file() return # Try error handling to create the parent directory if we need to try: _open_and_write_file() except NoSuchFile: # Try to create the parent directory, and then go back to # writing the file parent_dir = os.path.dirname(abspath) self._mkdir(parent_dir, dir_mode) _open_and_write_file() def put_file_non_atomic( self, relpath, f, mode=None, create_parent_dir=False, dir_mode=None ): """Copy the file-like object into the target location. This function is not strictly safe to use. It is only meant to be used when you already know that the target does not exist. It is not safe, because it will open and truncate the remote file. So there may be a time when the file has invalid contents. :param relpath: The remote location to put the contents. :param f: File-like object. :param mode: Possible access permissions for new file. None means do not set remote permissions. :param create_parent_dir: If we cannot create the target file because the parent directory does not exist, go ahead and create it, and then try again. """ def writer(fout): self._pump(f, fout) self._put_non_atomic_helper( relpath, writer, mode=mode, create_parent_dir=create_parent_dir, dir_mode=dir_mode, ) def put_bytes_non_atomic( self, relpath: str, raw_bytes: bytes, mode=None, create_parent_dir=False, dir_mode=None, ): """Write bytes to a file non-atomically. This is not safe if the target already exists as it will truncate it. Args: relpath: Path relative to transport root. raw_bytes: Bytes to write. mode: File permissions. create_parent_dir: Whether to create parent directory if needed. dir_mode: Permissions for created parent directories. Raises: TypeError: If raw_bytes is not bytes. """ if not isinstance(raw_bytes, bytes): raise TypeError(f"raw_bytes must be a plain string, not {type(raw_bytes)}") def writer(fout): fout.write(raw_bytes) self._put_non_atomic_helper( relpath, writer, mode=mode, create_parent_dir=create_parent_dir, dir_mode=dir_mode, ) def iter_files_recursive(self): """Walk the relative paths of all files in this transport.""" # progress is handled by list_dir queue = list(self.list_dir(".")) while queue: relpath = queue.pop(0) st = self.stat(relpath) if stat.S_ISDIR(st.st_mode): for i, basename in enumerate(self.list_dir(relpath)): queue.insert(i, relpath + "/" + basename) else: yield relpath def _mkdir(self, abspath, mode=None): local_mode = 511 if mode is None else mode try: self._report_activity(len(abspath), "write") self._get_sftp().mkdir(abspath, local_mode) self._report_activity(1, "read") if mode is not None: # chmod a dir through sftp will erase any sgid bit set # on the server side. So, if the bit mode are already # set, avoid the chmod. If the mode is not fine but # the sgid bit is set, report a warning to the user # with the umask fix. stat = self._get_sftp().lstat(abspath) mode = mode & 0o777 # can't set special bits anyway if mode != stat.st_mode & 0o777: if stat.st_mode & 0o6000: logger.warning( f"About to chmod {abspath} over sftp, which will result" " in its suid or sgid bits being cleared. If" " you want to preserve those bits, change your " f" environment on the server to use umask 0{0o777 - mode:03o}." ) self._get_sftp().chmod(abspath, mode=mode) except (SFTPError, OSError) as e: self._translate_io_exception( e, abspath, ": unable to mkdir", failure_exc=FileExists ) def mkdir(self, relpath, mode=None): """Create a directory at the given path.""" self._mkdir(self._remote_path(relpath), mode=mode) def open_write_stream(self, relpath, mode=None): """See Transport.open_write_stream.""" # initialise the file to zero-length # this is three round trips, but we don't use this # api more than once per write_group at the moment so # it is a tolerable overhead. Better would be to truncate # the file after opening. RBC 20070805 self.put_bytes_non_atomic(relpath, b"", mode) abspath = self._remote_path(relpath) # TODO: jam 20060816 paramiko doesn't publicly expose a way to # set the file mode at create time. If it does, use it. # But for now, we just chmod later anyway. handle = None try: handle = self._get_sftp().file(abspath, mode="wb") except (SFTPError, OSError) as e: self._translate_io_exception(e, abspath, ": unable to open") _file_streams[self.abspath(relpath)] = handle return FileFileStream(self, relpath, handle) def _translate_io_exception(self, e, path, more_info="", failure_exc=PathError): """Translate a paramiko or IOError into a friendlier exception. :param e: The original exception :param path: The path in question when the error is raised :param more_info: Extra information that can be included, such as what was going on :param failure_exc: Paramiko has the super fun ability to raise completely opaque errors that just set "e.args = ('Failure',)" with no more information. If this parameter is set, it defines the exception to raise in these cases. """ # paramiko seems to generate detailless errors. self._translate_error(e, path, raise_generic=False) if getattr(e, "args", None) is not None: if e.args == ("No such file or directory",) or e.args == ("No such file",): raise NoSuchFile(path, str(e) + more_info) if e.args == ("mkdir failed",) or e.args[0].startswith( "syserr: File exists" ): raise FileExists(path, str(e) + more_info) # strange but true, for the paramiko server. if e.args == ("Failure",): raise failure_exc(path, str(e) + more_info) # Can be something like args = ('Directory not empty: # '/srv/example.com/blah...: ' # [Errno 39] Directory not empty',) if ( e.args[0].startswith("Directory not empty: ") or getattr(e, "errno", None) == errno.ENOTEMPTY ): raise errors.DirectoryNotEmpty(path, str(e)) if e.args == ("Operation unsupported",): raise TransportNotPossible() logger.debug("Raising exception with args %s", e.args) if getattr(e, "errno", None) is not None: logger.debug("Raising exception with errno %s", e.errno) raise e def append_file(self, relpath, f, mode=None): """Append the text in the file-like object into the final location. """ try: path = self._remote_path(relpath) fout = self._get_sftp().file(path, "ab") if mode is not None: self._get_sftp().chmod(path, mode) result = fout.tell() self._pump(f, fout) return result except (OSError, SFTPError) as e: self._translate_io_exception(e, relpath, ": unable to append") def rename(self, rel_from, rel_to): """Rename without special overwriting.""" try: self._get_sftp().rename( self._remote_path(rel_from), self._remote_path(rel_to) ) except (OSError, SFTPError) as e: self._translate_io_exception( e, rel_from, f": unable to rename to {rel_to!r}" ) def _rename_and_overwrite(self, abs_from, abs_to): """Do a fancy rename on the remote server. Using the implementation provided by osutils. """ try: sftp = self._get_sftp() fancy_rename( abs_from, abs_to, rename_func=sftp.rename, unlink_func=sftp.remove ) except (OSError, SFTPError) as e: self._translate_io_exception( e, abs_from, f": unable to rename to {abs_to!r}" ) def move(self, rel_from, rel_to): """Move the item at rel_from to the location at rel_to.""" path_from = self._remote_path(rel_from) path_to = self._remote_path(rel_to) self._rename_and_overwrite(path_from, path_to) def delete(self, relpath): """Delete the item at relpath.""" path = self._remote_path(relpath) try: self._get_sftp().remove(path) except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": unable to delete") def external_url(self): """See dromedary.Transport.external_url.""" # the external path for SFTP is the base return self.base def listable(self): """Return True if this store supports listing.""" return True def list_dir(self, relpath): """Return a list of all files at the given location.""" # does anything actually use this? # -- Unknown # This is at least used by copy_tree for remote upgrades. # -- David Allouche 2006-08-11 path = self._remote_path(relpath) try: entries = self._get_sftp().listdir(path) self._report_activity(sum(map(len, entries)), "read") except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": failed to list_dir") return [urlutils.escape(entry) for entry in entries] def rmdir(self, relpath): """See Transport.rmdir.""" path = self._remote_path(relpath) try: return self._get_sftp().rmdir(path) except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": failed to rmdir") def stat(self, relpath): """Return the stat information for a file.""" path = self._remote_path(relpath) try: return self._get_sftp().lstat(path) except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": unable to stat") def readlink(self, relpath): """See Transport.readlink.""" path = self._remote_path(relpath) try: return self._get_sftp().readlink(self._remote_path(path)) except (OSError, SFTPError) as e: self._translate_io_exception(e, path, ": unable to readlink") def symlink(self, source, link_name): """See Transport.symlink.""" try: conn = self._get_sftp() conn.symlink(source, self._remote_path(link_name)) except (OSError, SFTPError) as e: self._translate_io_exception( e, link_name, f": unable to create symlink to {source!r}" ) def lock_read(self, relpath): """Lock the given file for shared (read) access. :return: A lock object, which has an unlock() member function. """ # FIXME: there should be something clever i can do here... class BogusLock: def __init__(self, path): self.path = path def unlock(self): pass def __exit__(self, exc_type, exc_val, exc_tb): return False def __enter__(self): pass return BogusLock(relpath) def lock_write(self, relpath): """Lock the given file for exclusive (write) access. WARNING: many transports do not support this, so trying avoid using it. :return: A lock object, which has an unlock() member function """ # This is a little bit bogus, but basically, we create a file # which should not already exist, and if it does, we assume # that there is a lock, and if it doesn't, the we assume # that we have taken the lock. return SFTPLock(relpath, self) def _sftp_open_exclusive(self, abspath, mode=None): """Open a remote path exclusively. SFTP supports O_EXCL (SFTP_FLAG_EXCL), which fails if the file already exists. However it does not expose this at the higher level of SFTPClient.open(), so we have to sneak away with it. WARNING: This breaks the SFTPClient abstraction, so it could easily break against an updated version of paramiko. :param abspath: The remote absolute path where the file should be opened :param mode: The mode permissions bits for the new file """ attr = _sftp_rs.SFTPAttributes() if mode is not None: attr.st_mode = mode | stat.S_IFREG else: # Apply the local umask to the default 0o666 so file permissions # follow the same convention as ordinary file creation. from dromedary.osutils import get_umask attr.st_mode = stat.S_IFREG | (0o666 & ~get_umask()) try: return self._get_sftp().open( abspath, attr, write=True, create=True, excl=True, truncate=True ) except (SFTPError, OSError) as e: self._translate_io_exception( e, abspath, ": unable to open", failure_exc=FileExists ) def _can_roundtrip_unix_modebits(self): return sys.platform != "win32" def get_test_permutations(): """Return the permutations to be used in testing.""" import importlib.util if importlib.util.find_spec("paramiko") is None: raise ParamikoNotPresent("paramiko not installed") from dromedary.tests import stub_sftp return [ (SFTPTransport, stub_sftp.SFTPAbsoluteServer), (SFTPTransport, stub_sftp.SFTPHomeDirServer), (SFTPTransport, stub_sftp.SFTPSiblingAbsoluteServer), ] dromedary-0.1.1/src/brokenrename.rs000064400000000000000000000065751046102023000154330ustar 00000000000000//! BrokenRename Transport decorator, ported from dromedary/brokenrename.py. //! //! A transport that fails to detect clashing renames: if the destination //! exists, the rename is silently absorbed rather than raising an error. use crate::{Result, Transport, UrlFragment}; use url::Url; pub struct BrokenRenameTransport { inner: Box, base: Url, } impl BrokenRenameTransport { pub const PREFIX: &'static str = "brokenrename+"; pub fn new(inner: Box) -> Self { let base = crate::decorator::prefixed_base(Self::PREFIX, inner.as_ref()); Self { inner, base } } } impl std::fmt::Debug for BrokenRenameTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "BrokenRenameTransport({})", self.base) } } impl Transport for BrokenRenameTransport { crate::fwd_external_url!(inner); crate::fwd_can_roundtrip_unix_modebits!(inner); crate::fwd_is_readonly!(inner); crate::fwd_listable!(inner); crate::fwd_get!(inner); crate::fwd_has!(inner); crate::fwd_stat!(inner); crate::fwd_clone!(inner); crate::fwd_abspath!(inner); crate::fwd_relpath!(inner); crate::fwd_put_file!(inner); crate::fwd_mkdir!(inner); crate::fwd_delete!(inner); crate::fwd_rmdir!(inner); crate::fwd_set_segment_parameter!(inner); crate::fwd_get_segment_parameters!(inner); crate::fwd_append_file!(inner); crate::fwd_readlink!(inner); crate::fwd_hardlink!(inner); crate::fwd_symlink!(inner); crate::fwd_iter_files_recursive!(inner); crate::fwd_open_write_stream!(inner); crate::fwd_delete_tree!(inner); crate::fwd_move!(inner); crate::fwd_list_dir!(inner); crate::fwd_lock_read!(inner); crate::fwd_lock_write!(inner); crate::fwd_local_abspath!(inner); crate::fwd_get_smart_medium!(inner); crate::fwd_copy!(inner); fn base(&self) -> Url { self.base.clone() } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { match self.inner.rename(rel_from, rel_to) { Ok(()) => Ok(()), // Absorb clashes silently — that's the whole point. Err(crate::Error::FileExists(_)) | Err(crate::Error::DirectoryNotEmptyError(_)) => { Ok(()) } Err(e) => Err(e), } } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn wrap() -> BrokenRenameTransport { let mem = MemoryTransport::new("memory:///").unwrap(); mem.put_bytes("a", b"A", None).unwrap(); mem.put_bytes("b", b"B", None).unwrap(); BrokenRenameTransport::new(Box::new(mem)) } #[test] fn base_prefix() { assert!(wrap().base().as_str().starts_with("brokenrename+")); } #[test] fn ok_rename_still_works() { let t = wrap(); t.rename("a", "c").unwrap(); assert_eq!(t.has("a").unwrap(), false); assert_eq!(t.get_bytes("c").unwrap(), b"A"); } #[test] fn clashing_rename_is_absorbed() { let t = wrap(); // Renaming a over existing b would normally raise FileExists. t.rename("a", "b").unwrap(); // Both files should still exist because the rename was absorbed. assert_eq!(t.get_bytes("a").unwrap(), b"A"); assert_eq!(t.get_bytes("b").unwrap(), b"B"); } } dromedary-0.1.1/src/chroot.rs000064400000000000000000000051311046102023000142440ustar 00000000000000//! Chroot Transport, ported from dromedary/chroot.py. //! //! A chroot is a [`PathFilteringTransport`](crate::pathfilter::PathFilteringTransport) //! with no user filter function: the server-root rebase that `pathfilter` //! performs is enough to prevent `..` sequences from escaping the backing //! transport's root. use crate::pathfilter::PathFilteringTransport; use crate::{Result, Transport}; /// Construct a chroot transport wrapping `backing`, exposed under `scheme` /// (e.g. "chroot-42:///") with `base_path` (must start with `/`) as the /// chroot root within the backing transport. pub fn new_chroot( backing: Box, scheme: impl Into, base_path: impl Into, ) -> Result { PathFilteringTransport::new(backing, scheme, base_path, None) } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; use crate::Error; fn chroot_at() -> PathFilteringTransport { // Mirror the Python setup: backing is already rooted inside the jail, // so escapes via `..` resolve to paths that don't exist on the // backing transport. let mem = MemoryTransport::new("memory:///").unwrap(); mem.mkdir("jail", None).unwrap(); mem.put_bytes("jail/inside", b"ok", None).unwrap(); mem.put_bytes("outside", b"secret", None).unwrap(); let jail = mem.clone(Some("jail")).unwrap(); new_chroot(jail, "chroot-1:///", "/").unwrap() } #[test] fn reads_inside_jail() { let t = chroot_at(); assert_eq!(t.get_bytes("inside").unwrap(), b"ok"); } #[test] fn dotdot_cannot_escape_chroot() { let t = chroot_at(); match t.get_bytes("../outside") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn deeper_dotdot_cannot_escape_chroot() { let t = chroot_at(); match t.get_bytes("../../outside") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn absolute_path_cannot_escape_chroot() { let t = chroot_at(); match t.get_bytes("/outside") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn mkdir_and_delete_round_trip() { let t = chroot_at(); t.mkdir("new", None).unwrap(); t.put_bytes("new/f", b"x", None).unwrap(); assert_eq!(t.get_bytes("new/f").unwrap(), b"x"); } } dromedary-0.1.1/src/decorator.rs000064400000000000000000000241711046102023000147350ustar 00000000000000//! Shared forwarding helpers for Transport decorators, matching //! dromedary/decorator.py. //! //! `forward_transport_all!($field)` emits a default forwarding impl for //! every Transport method. Each individual forwarder is also exposed as //! `fwd_!` so decorators that want to override a few methods can //! invoke only the forwarders they need instead of relying on skip //! lists. /// Build a decorator base URL by prefixing the inner transport's base. /// `prefix` should include the trailing `+`, matching Python's /// `_get_url_prefix()` convention (e.g. "fakenfs+"). pub fn prefixed_base(prefix: &str, inner: &dyn crate::Transport) -> ::url::Url { let inner_base = inner.base(); let url = format!("{}{}", prefix, inner_base); ::url::Url::parse(&url).unwrap_or(inner_base) } #[macro_export] macro_rules! fwd_external_url { ($field:ident) => { fn external_url(&self) -> $crate::Result<::url::Url> { self.$field.external_url() } }; } #[macro_export] macro_rules! fwd_can_roundtrip_unix_modebits { ($field:ident) => { fn can_roundtrip_unix_modebits(&self) -> bool { self.$field.can_roundtrip_unix_modebits() } }; } #[macro_export] macro_rules! fwd_is_readonly { ($field:ident) => { fn is_readonly(&self) -> bool { self.$field.is_readonly() } }; } #[macro_export] macro_rules! fwd_listable { ($field:ident) => { fn listable(&self) -> bool { self.$field.listable() } }; } #[macro_export] macro_rules! fwd_get { ($field:ident) => { fn get( &self, relpath: &$crate::UrlFragment, ) -> $crate::Result> { self.$field.get(relpath) } }; } #[macro_export] macro_rules! fwd_has { ($field:ident) => { fn has(&self, relpath: &$crate::UrlFragment) -> $crate::Result { self.$field.has(relpath) } }; } #[macro_export] macro_rules! fwd_stat { ($field:ident) => { fn stat(&self, relpath: &$crate::UrlFragment) -> $crate::Result<$crate::Stat> { self.$field.stat(relpath) } }; } #[macro_export] macro_rules! fwd_clone { ($field:ident) => { fn clone( &self, offset: Option<&$crate::UrlFragment>, ) -> $crate::Result> { self.$field.clone(offset) } }; } #[macro_export] macro_rules! fwd_abspath { ($field:ident) => { fn abspath(&self, relpath: &$crate::UrlFragment) -> $crate::Result<::url::Url> { self.$field.abspath(relpath) } }; } #[macro_export] macro_rules! fwd_relpath { ($field:ident) => { fn relpath(&self, abspath: &::url::Url) -> $crate::Result { self.$field.relpath(abspath) } }; } #[macro_export] macro_rules! fwd_put_file { ($field:ident) => { fn put_file( &self, relpath: &$crate::UrlFragment, f: &mut dyn ::std::io::Read, permissions: Option<::std::fs::Permissions>, ) -> $crate::Result { self.$field.put_file(relpath, f, permissions) } }; } #[macro_export] macro_rules! fwd_mkdir { ($field:ident) => { fn mkdir( &self, relpath: &$crate::UrlFragment, permissions: Option<::std::fs::Permissions>, ) -> $crate::Result<()> { self.$field.mkdir(relpath, permissions) } }; } #[macro_export] macro_rules! fwd_delete { ($field:ident) => { fn delete(&self, relpath: &$crate::UrlFragment) -> $crate::Result<()> { self.$field.delete(relpath) } }; } #[macro_export] macro_rules! fwd_rmdir { ($field:ident) => { fn rmdir(&self, relpath: &$crate::UrlFragment) -> $crate::Result<()> { self.$field.rmdir(relpath) } }; } #[macro_export] macro_rules! fwd_rename { ($field:ident) => { fn rename( &self, rel_from: &$crate::UrlFragment, rel_to: &$crate::UrlFragment, ) -> $crate::Result<()> { self.$field.rename(rel_from, rel_to) } }; } #[macro_export] macro_rules! fwd_set_segment_parameter { ($field:ident) => { fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> $crate::Result<()> { self.$field.set_segment_parameter(key, value) } }; } #[macro_export] macro_rules! fwd_get_segment_parameters { ($field:ident) => { fn get_segment_parameters( &self, ) -> $crate::Result<::std::collections::HashMap> { self.$field.get_segment_parameters() } }; } #[macro_export] macro_rules! fwd_append_file { ($field:ident) => { fn append_file( &self, relpath: &$crate::UrlFragment, f: &mut dyn ::std::io::Read, permissions: Option<::std::fs::Permissions>, ) -> $crate::Result { self.$field.append_file(relpath, f, permissions) } }; } #[macro_export] macro_rules! fwd_readlink { ($field:ident) => { fn readlink(&self, relpath: &$crate::UrlFragment) -> $crate::Result { self.$field.readlink(relpath) } }; } #[macro_export] macro_rules! fwd_hardlink { ($field:ident) => { fn hardlink( &self, rel_from: &$crate::UrlFragment, rel_to: &$crate::UrlFragment, ) -> $crate::Result<()> { self.$field.hardlink(rel_from, rel_to) } }; } #[macro_export] macro_rules! fwd_symlink { ($field:ident) => { fn symlink( &self, rel_from: &$crate::UrlFragment, rel_to: &$crate::UrlFragment, ) -> $crate::Result<()> { self.$field.symlink(rel_from, rel_to) } }; } #[macro_export] macro_rules! fwd_iter_files_recursive { ($field:ident) => { fn iter_files_recursive(&self) -> Box>> { self.$field.iter_files_recursive() } }; } #[macro_export] macro_rules! fwd_open_write_stream { ($field:ident) => { fn open_write_stream( &self, relpath: &$crate::UrlFragment, permissions: Option<::std::fs::Permissions>, ) -> $crate::Result> { self.$field.open_write_stream(relpath, permissions) } }; } #[macro_export] macro_rules! fwd_delete_tree { ($field:ident) => { fn delete_tree(&self, relpath: &$crate::UrlFragment) -> $crate::Result<()> { self.$field.delete_tree(relpath) } }; } #[macro_export] macro_rules! fwd_move { ($field:ident) => { fn r#move( &self, rel_from: &$crate::UrlFragment, rel_to: &$crate::UrlFragment, ) -> $crate::Result<()> { self.$field.r#move(rel_from, rel_to) } }; } #[macro_export] macro_rules! fwd_list_dir { ($field:ident) => { fn list_dir( &self, relpath: &$crate::UrlFragment, ) -> Box>> { self.$field.list_dir(relpath) } }; } #[macro_export] macro_rules! fwd_lock_read { ($field:ident) => { fn lock_read( &self, relpath: &$crate::UrlFragment, ) -> $crate::Result> { self.$field.lock_read(relpath) } }; } #[macro_export] macro_rules! fwd_lock_write { ($field:ident) => { fn lock_write( &self, relpath: &$crate::UrlFragment, ) -> $crate::Result> { self.$field.lock_write(relpath) } }; } #[macro_export] macro_rules! fwd_local_abspath { ($field:ident) => { fn local_abspath( &self, relpath: &$crate::UrlFragment, ) -> $crate::Result<::std::path::PathBuf> { self.$field.local_abspath(relpath) } }; } #[macro_export] macro_rules! fwd_get_smart_medium { ($field:ident) => { fn get_smart_medium(&self) -> $crate::Result> { self.$field.get_smart_medium() } }; } #[macro_export] macro_rules! fwd_copy { ($field:ident) => { fn copy( &self, rel_from: &$crate::UrlFragment, rel_to: &$crate::UrlFragment, ) -> $crate::Result<()> { self.$field.copy(rel_from, rel_to) } }; } /// Forward every Transport method to `self.$field`. The caller must still /// define `fn base(&self) -> Url`. Use this when no methods need to be /// overridden; decorators that override a few methods should invoke the /// individual `fwd_*!` macros instead. #[macro_export] macro_rules! fwd_all { ($field:ident) => { $crate::fwd_external_url!($field); $crate::fwd_can_roundtrip_unix_modebits!($field); $crate::fwd_is_readonly!($field); $crate::fwd_listable!($field); $crate::fwd_get!($field); $crate::fwd_has!($field); $crate::fwd_stat!($field); $crate::fwd_clone!($field); $crate::fwd_abspath!($field); $crate::fwd_relpath!($field); $crate::fwd_put_file!($field); $crate::fwd_mkdir!($field); $crate::fwd_delete!($field); $crate::fwd_rmdir!($field); $crate::fwd_rename!($field); $crate::fwd_set_segment_parameter!($field); $crate::fwd_get_segment_parameters!($field); $crate::fwd_append_file!($field); $crate::fwd_readlink!($field); $crate::fwd_hardlink!($field); $crate::fwd_symlink!($field); $crate::fwd_iter_files_recursive!($field); $crate::fwd_open_write_stream!($field); $crate::fwd_delete_tree!($field); $crate::fwd_move!($field); $crate::fwd_list_dir!($field); $crate::fwd_lock_read!($field); $crate::fwd_lock_write!($field); $crate::fwd_local_abspath!($field); $crate::fwd_get_smart_medium!($field); $crate::fwd_copy!($field); }; } dromedary-0.1.1/src/fakenfs.rs000064400000000000000000000111541046102023000143650ustar 00000000000000//! FakeNFS Transport decorator, ported from dromedary/fakenfs.py. //! //! Adapts any Transport to behave like NFS for testing: rename against a //! non-empty target directory raises ResourceBusy, and deleting a file //! whose basename starts with ".nfs" raises ResourceBusy. use crate::{Error, Result, Stat, Transport, UrlFragment}; use url::Url; pub struct FakeNfsTransport { inner: Box, base: Url, } impl FakeNfsTransport { pub const PREFIX: &'static str = "fakenfs+"; pub fn new(inner: Box) -> Self { let base = crate::decorator::prefixed_base(Self::PREFIX, inner.as_ref()); Self { inner, base } } } impl std::fmt::Debug for FakeNfsTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "FakeNfsTransport({})", self.base) } } fn basename(path: &str) -> &str { match path.rsplit_once('/') { Some((_, tail)) => tail, None => path, } } impl Transport for FakeNfsTransport { crate::fwd_external_url!(inner); crate::fwd_can_roundtrip_unix_modebits!(inner); crate::fwd_is_readonly!(inner); crate::fwd_listable!(inner); crate::fwd_get!(inner); crate::fwd_has!(inner); crate::fwd_stat!(inner); crate::fwd_clone!(inner); crate::fwd_abspath!(inner); crate::fwd_relpath!(inner); crate::fwd_put_file!(inner); crate::fwd_mkdir!(inner); crate::fwd_rmdir!(inner); crate::fwd_set_segment_parameter!(inner); crate::fwd_get_segment_parameters!(inner); crate::fwd_append_file!(inner); crate::fwd_readlink!(inner); crate::fwd_hardlink!(inner); crate::fwd_symlink!(inner); crate::fwd_iter_files_recursive!(inner); crate::fwd_open_write_stream!(inner); crate::fwd_delete_tree!(inner); crate::fwd_move!(inner); crate::fwd_list_dir!(inner); crate::fwd_lock_read!(inner); crate::fwd_lock_write!(inner); crate::fwd_local_abspath!(inner); crate::fwd_get_smart_medium!(inner); crate::fwd_copy!(inner); fn base(&self) -> Url { self.base.clone() } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { match self.inner.rename(rel_from, rel_to) { Ok(()) => Ok(()), Err(e @ Error::DirectoryNotEmptyError(_)) | Err(e @ Error::FileExists(_)) => { match self.inner.stat(rel_to) { Ok(Stat { kind, .. }) if kind == crate::FileKind::Dir => { Err(Error::ResourceBusy(Some(rel_to.to_string()))) } _ => Err(e), } } Err(e) => Err(e), } } fn delete(&self, relpath: &UrlFragment) -> Result<()> { if basename(relpath).starts_with(".nfs") { return Err(Error::ResourceBusy(Some(relpath.to_string()))); } self.inner.delete(relpath) } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn wrap() -> FakeNfsTransport { let mem = MemoryTransport::new("memory:///").unwrap(); mem.put_bytes("regular", b"x", None).unwrap(); mem.put_bytes(".nfs1234", b"busy", None).unwrap(); mem.mkdir("dir1", None).unwrap(); mem.mkdir("dir2", None).unwrap(); mem.put_bytes("f1", b"a", None).unwrap(); mem.put_bytes("f2", b"b", None).unwrap(); FakeNfsTransport::new(Box::new(mem)) } #[test] fn base_has_fakenfs_prefix() { assert!(wrap().base().as_str().starts_with("fakenfs+")); } #[test] fn regular_delete_passes_through() { wrap().delete("regular").unwrap(); } #[test] fn dotnfs_delete_is_busy() { match wrap().delete(".nfs1234") { Err(Error::ResourceBusy(_)) => {} other => panic!("expected ResourceBusy, got {:?}", other), } } #[test] fn rename_dir_over_dir_becomes_busy() { let t = wrap(); match t.rename("dir1", "dir2") { Err(Error::ResourceBusy(_)) => {} other => panic!("expected ResourceBusy, got {:?}", other), } } #[test] fn rename_file_over_file_propagates_original_error() { // Destination is a file, so the translator's stat check falls through // and the original FileExists is re-raised. let t = wrap(); match t.rename("f1", "f2") { Err(Error::FileExists(_)) => {} other => panic!("expected FileExists, got {:?}", other), } } #[test] fn reads_pass_through() { let t = wrap(); assert_eq!(t.get_bytes("regular").unwrap(), b"x"); } } dromedary-0.1.1/src/fakevfat.rs000064400000000000000000000107521046102023000145420ustar 00000000000000//! FakeVFAT Transport decorator, ported from dromedary/fakevfat.py. //! //! Simulates VFAT restrictions: filenames are squashed to lowercase, and //! names containing any of `?*:;<>` are rejected. Only a subset of //! Transport methods route through the squash; others forward unchanged. use crate::{Error, Result, Transport, UrlFragment}; use std::fs::Permissions; use url::Url; pub struct FakeVfatTransport { inner: Box, base: Url, } impl FakeVfatTransport { pub const PREFIX: &'static str = "vfat+"; pub fn new(inner: Box) -> Self { let base = crate::decorator::prefixed_base(Self::PREFIX, inner.as_ref()); Self { inner, base } } fn squash_name(name: &str) -> Result { if name.contains(|c: char| matches!(c, '?' | '*' | ':' | ';' | '<' | '>')) { return Err(Error::PathNotChild); } Ok(name.to_lowercase()) } } impl std::fmt::Debug for FakeVfatTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "FakeVfatTransport({})", self.base) } } impl Transport for FakeVfatTransport { crate::fwd_external_url!(inner); crate::fwd_is_readonly!(inner); crate::fwd_listable!(inner); crate::fwd_stat!(inner); crate::fwd_clone!(inner); crate::fwd_abspath!(inner); crate::fwd_relpath!(inner); crate::fwd_delete!(inner); crate::fwd_rmdir!(inner); crate::fwd_rename!(inner); crate::fwd_set_segment_parameter!(inner); crate::fwd_get_segment_parameters!(inner); crate::fwd_append_file!(inner); crate::fwd_readlink!(inner); crate::fwd_hardlink!(inner); crate::fwd_symlink!(inner); crate::fwd_iter_files_recursive!(inner); crate::fwd_open_write_stream!(inner); crate::fwd_delete_tree!(inner); crate::fwd_move!(inner); crate::fwd_list_dir!(inner); crate::fwd_lock_read!(inner); crate::fwd_lock_write!(inner); crate::fwd_local_abspath!(inner); crate::fwd_get_smart_medium!(inner); crate::fwd_copy!(inner); fn base(&self) -> Url { self.base.clone() } fn can_roundtrip_unix_modebits(&self) -> bool { false } fn get(&self, relpath: &UrlFragment) -> Result> { self.inner.get(&Self::squash_name(relpath)?) } fn has(&self, relpath: &UrlFragment) -> Result { self.inner.has(&Self::squash_name(relpath)?) } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn std::io::Read, permissions: Option, ) -> Result { self.inner .put_file(&Self::squash_name(relpath)?, f, permissions) } fn mkdir(&self, relpath: &UrlFragment, _permissions: Option) -> Result<()> { // Python hard-codes 0o755 for VFAT mkdir. #[cfg(unix)] let perms = { use std::os::unix::fs::PermissionsExt; Some(Permissions::from_mode(0o755)) }; #[cfg(not(unix))] let perms: Option = None; self.inner.mkdir(&Self::squash_name(relpath)?, perms) } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn wrap() -> FakeVfatTransport { let mem = MemoryTransport::new("memory:///").unwrap(); // Pre-seed lowercase names so squashed reads find them. mem.put_bytes("readme", b"hi", None).unwrap(); FakeVfatTransport::new(Box::new(mem)) } #[test] fn base_prefix() { assert!(wrap().base().as_str().starts_with("vfat+")); } #[test] fn uppercase_get_squashes_to_lowercase() { assert_eq!(wrap().get_bytes("README").unwrap(), b"hi"); } #[test] fn illegal_character_rejected() { match wrap().put_bytes("bad:name", b"x", None) { Err(Error::PathNotChild) => {} other => panic!("expected PathNotChild, got {:?}", other), } } #[test] fn put_and_get_round_trip_lowercase() { let t = wrap(); t.put_bytes("HELLO", b"world", None).unwrap(); assert_eq!(t.get_bytes("hello").unwrap(), b"world"); assert_eq!(t.get_bytes("Hello").unwrap(), b"world"); } #[test] fn roundtrip_unix_modebits_false() { assert_eq!(wrap().can_roundtrip_unix_modebits(), false); } #[test] fn mkdir_squashes_name() { let t = wrap(); t.mkdir("NewDir", None).unwrap(); assert_eq!(t.has("newdir").unwrap(), true); } } dromedary-0.1.1/src/fcntl-locks.rs000064400000000000000000000237141046102023000151740ustar 00000000000000use crate::lock::{FileLock, Lock, LockError}; use lazy_static::lazy_static; use log::debug; use nix::fcntl::{fcntl, FcntlArg}; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; fn open(filename: &Path, options: &OpenOptions) -> std::result::Result<(PathBuf, File), LockError> { let filename = crate::osutils::path::realpath(filename)?; match options.open(&filename) { Ok(f) => Ok((filename, f)), Err(e) => match e.kind() { std::io::ErrorKind::PermissionDenied => Err(LockError::Failed(filename, e.to_string())), std::io::ErrorKind::NotFound => { // Maybe this is an old branch (before 2005)? debug!( "trying to create missing lock {}", filename.to_string_lossy() ); let f = OpenOptions::new() .create(true) .write(true) .read(true) .open(&filename)?; Ok((filename, f)) } _ => Err(e.into()), }, } } lazy_static! { static ref OPEN_WRITE_LOCKS: std::sync::Mutex> = std::sync::Mutex::new(HashSet::new()); static ref OPEN_READ_LOCKS: std::sync::Mutex> = std::sync::Mutex::new(HashMap::new()); } pub struct WriteLock { filename: PathBuf, f: File, } impl WriteLock { pub fn new(filename: &Path, strict_locks: bool) -> Result { let filename = crate::osutils::path::realpath(filename)?; if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { return Err(LockError::Contention(filename)); } if OPEN_READ_LOCKS.lock().unwrap().contains_key(&filename) { if strict_locks { return Err(LockError::Contention(filename)); } else { debug!( "Write lock taken w/ an open read lock on: {}", filename.to_string_lossy() ); } } let (filename, f) = open( filename.as_path(), OpenOptions::new().read(true).write(true), )?; OPEN_WRITE_LOCKS.lock().unwrap().insert(filename.clone()); let flock = nix::libc::flock { l_type: nix::libc::F_WRLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; match fcntl(&f, FcntlArg::F_SETLK(&flock)) { Ok(_) => Ok(WriteLock { filename, f }), Err(e) => { if e == nix::errno::Errno::EAGAIN || e == nix::errno::Errno::EACCES { let flock = nix::libc::flock { l_type: nix::libc::F_UNLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; let _ = fcntl(&f, FcntlArg::F_SETLK(&flock)); } // we should be more precise about whats a locking // error and whats a random-other error Err(LockError::Contention(filename)) } } } } impl Lock for WriteLock { fn unlock(&mut self) -> Result<(), LockError> { OPEN_WRITE_LOCKS.lock().unwrap().remove(&self.filename); let flock = nix::libc::flock { l_type: nix::libc::F_UNLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; let _ = fcntl(&self.f, FcntlArg::F_SETLK(&flock)); Ok(()) } } impl FileLock for WriteLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } pub struct ReadLock { filename: PathBuf, f: File, } impl ReadLock { pub fn new(filename: &Path, strict_locks: bool) -> std::result::Result { let filename = crate::osutils::path::realpath(filename)?; if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { if strict_locks { return Err(LockError::Contention(filename)); } else { debug!( "Read lock taken w/ an open write lock on: {}", filename.to_string_lossy() ); } } OPEN_READ_LOCKS .lock() .unwrap() .entry(filename.clone()) .and_modify(|count| *count += 1) .or_insert(1); let (filename, f) = open(&filename, OpenOptions::new().read(true))?; let flock = nix::libc::flock { l_type: nix::libc::F_RDLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; match fcntl(&f, FcntlArg::F_SETLK(&flock)) { Ok(_) => {} Err(_e) => { // we should be more precise about whats a locking // error and whats a random-other error return Err(LockError::Contention(filename)); } } Ok(ReadLock { filename, f }) } /// Try to grab a write lock on the file. /// /// On platforms that support it, this will upgrade to a write lock /// without unlocking the file. /// Otherwise, this will release the read lock, and try to acquire a /// write lock. /// /// Returns: A token which can be used to switch back to a read lock. pub fn temporary_write_lock( self, ) -> std::result::Result { if OPEN_WRITE_LOCKS.lock().unwrap().contains(&self.filename) { panic!("file already locked: {}", self.filename.to_string_lossy()); } TemporaryWriteLock::new(self) } } impl Lock for ReadLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { match OPEN_READ_LOCKS.lock().unwrap().entry(self.filename.clone()) { Entry::Occupied(mut entry) => { let count = entry.get_mut(); if *count == 1 { entry.remove(); } else { *count -= 1; } } Entry::Vacant(_) => panic!("no read lock on {}", self.filename.to_string_lossy()), } let flock = nix::libc::flock { l_type: nix::libc::F_UNLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; let _ = fcntl(&self.f, FcntlArg::F_SETLK(&flock)); Ok(()) } } impl FileLock for ReadLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } /// A token used when grabbing a temporary_write_lock. /// /// Call restore_read_lock() when you are done with the write lock. pub struct TemporaryWriteLock { read_lock: ReadLock, filename: PathBuf, f: File, } impl TemporaryWriteLock { pub fn new(read_lock: ReadLock) -> std::result::Result { let filename = read_lock.filename.clone(); if let Some(count) = OPEN_READ_LOCKS.lock().unwrap().get(&filename) { if *count > 1 { // Something else also has a read-lock, so we cannot grab a // write lock. return Err((read_lock, LockError::Contention(filename))); } } if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { panic!("file already locked: {}", filename.to_string_lossy()); } // See if we can open the file for writing. Another process might // have a read lock. We don't use self._open() because we don't want // to create the file if it exists. That would have already been // done by ReadLock let f = match OpenOptions::new() .write(true) .read(true) .create(true) .open(&filename) { Ok(f) => Ok(f), Err(e) => return Err((read_lock, e.into())), }?; // LOCK_NB will cause IOError to be raised if we can't grab a // lock right away. let flock = nix::libc::flock { l_type: nix::libc::F_RDLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; match fcntl(&f, FcntlArg::F_SETLK(&flock)) { Ok(_) => Ok(()), Err(_) => { return Err((read_lock, LockError::Contention(filename))); } }?; OPEN_WRITE_LOCKS.lock().unwrap().insert(filename.clone()); Ok(Self { read_lock, filename, f, }) } /// Restore the original ReadLock. pub fn restore_read_lock(self) -> ReadLock { // For fcntl, since we never released the read lock, just release // the write lock, and return the original lock. let flock = nix::libc::flock { l_type: nix::libc::F_UNLCK as i16, l_whence: nix::libc::SEEK_SET as i16, l_start: 0, l_len: 0, l_pid: 0, }; match fcntl(&self.f, FcntlArg::F_SETLK(&flock)) { Ok(_) => {} Err(e) => { debug!( "error unlocking file {}: {}", &self.filename.to_string_lossy(), e ); } } OPEN_WRITE_LOCKS.lock().unwrap().remove(&self.filename); self.read_lock } } impl FileLock for TemporaryWriteLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } dromedary-0.1.1/src/gio.rs000064400000000000000000000724201046102023000135310ustar 00000000000000//! GIO Transport, ported from dromedary/gio_transport.py. //! //! Wraps `gio::File` to expose the dromedary [`Transport`] trait over //! anything gvfs can mount: `gio+file://`, `gio+sftp://`, `gio+smb://`, //! `gio+dav://`, `gio+ftp://`, `gio+ssh://`, `gio+obex://`. //! //! `gio::File` is `!Send`/`!Sync`, so we never store one on the struct; //! every method reconstructs files via `gio::File::for_uri` from the //! `String` base URL and a relpath. This sidesteps the threading //! constraints `Transport` imposes (`Send + Sync`). //! //! Mounting volumes that need credentials currently isn't implemented — //! v1 only handles URLs that gvfs can already enumerate. See the TODO //! near `ensure_mounted` for the path forward. use crate::lock::{Lock, LockError}; use crate::urlutils::escape; use crate::{ Error, FileKind, ReadStream, Result, SmartMedium, Stat, Transport, UrlFragment, WriteStream, }; use ::gio::prelude::*; use ::gio::{FileCopyFlags, FileQueryInfoFlags, IOErrorEnum}; use std::collections::HashMap; use std::fs::Permissions; use std::io::{Cursor, Read}; use std::sync::mpsc; use std::thread; use url::Url; const GIO_BACKENDS: &[&str] = &["dav", "file", "ftp", "obex", "sftp", "ssh", "smb"]; /// A transport that proxies through a gvfs mount. pub struct GioTransport { /// Public dromedary base, including the `gio+` scheme prefix and a /// trailing slash. base: Url, /// URL stripped of the `gio+` prefix and any embedded credentials, /// suitable to pass to `gio::File::for_uri`. backend_url: String, } impl std::fmt::Debug for GioTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "GioTransport({})", self.base) } } impl GioTransport { pub fn new(base: &str) -> Result { let mut base = base.to_string(); if !base.ends_with('/') { base.push('/'); } let stripped = base .strip_prefix("gio+") .ok_or_else(|| Error::NotLocalUrl(base.clone()))?; let parsed = Url::parse(stripped).map_err(Error::from)?; if !GIO_BACKENDS.contains(&parsed.scheme()) { return Err(Error::UrlError(url::ParseError::IdnaError)); } // Reconstruct the backend URL with any embedded user/password // stripped — gvfs handles credentials via MountOperation, not // via the URL. let mut backend = parsed.clone(); let _ = backend.set_username(""); let _ = backend.set_password(None); let backend_url = backend.to_string(); let public_base = Url::parse(&base).map_err(Error::from)?; Ok(Self { base: public_base, backend_url, }) } fn child_url(&self, relpath: &UrlFragment) -> Result { // The backend URL ends with `/` so url::Url::join treats it as a // directory and resolves relpaths relative to it. An empty or // `.` relpath returns the directory itself. let base = Url::parse(&self.backend_url).map_err(Error::from)?; let trimmed = if relpath == "." || relpath.is_empty() { "" } else { relpath }; let joined = base.join(trimmed).map_err(Error::from)?; Ok(joined.to_string()) } fn file_for(&self, relpath: &UrlFragment) -> Result<::gio::File> { let url = self.child_url(relpath)?; Ok(::gio::File::for_uri(&url)) } /// Translate a gvfs error into the dromedary error vocabulary. fn translate(err: glib::Error, relpath: Option<&UrlFragment>) -> Error { let path = relpath.map(|p| p.to_string()); match err.kind::() { Some(IOErrorEnum::NotFound) => Error::NoSuchFile(path), Some(IOErrorEnum::Exists) => Error::FileExists(path), Some(IOErrorEnum::NotDirectory) => Error::NotADirectoryError(path), Some(IOErrorEnum::IsDirectory) => Error::IsADirectoryError(path), Some(IOErrorEnum::NotEmpty) => Error::DirectoryNotEmptyError(path), Some(IOErrorEnum::PermissionDenied) => Error::PermissionDenied(path), Some(IOErrorEnum::Busy) => Error::ResourceBusy(path), Some(IOErrorEnum::NotMounted) => Error::TransportNotPossible, Some(IOErrorEnum::ReadOnly) => Error::PermissionDenied(path), // Everything else folds into a generic IO error so the caller // gets *something* useful instead of a panic. _ => Error::Io(std::io::Error::other(err.to_string())), } } } struct GioReadStream(Cursor>); impl Read for GioReadStream { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.0.read(buf) } } impl std::io::Seek for GioReadStream { fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { self.0.seek(pos) } } impl ReadStream for GioReadStream {} /// Commands sent from the public `GioWriteStream` handle to the worker /// thread that owns the underlying `gio::FileOutputStream`. The worker /// is necessary because `FileOutputStream` is `!Send`, but `WriteStream` /// requires `Send + Sync`. enum WriterCmd { Write(Vec), Flush, Close, } /// Reply payload returned over a one-shot reply channel for each command. /// `glib::Error` is `!Send`, so any error is converted to a string here. type WriterReply = std::result::Result; /// Send+Sync handle to a writer thread that owns a gvfs output stream. struct GioWriteStream { tx: Option)>>, join: Option>, } impl GioWriteStream { fn spawn(url: String) -> Result { // The worker creates the output stream itself so the !Send // `gio::File` / `FileOutputStream` never crosses thread boundaries. // Open synchronously via a one-shot channel so we can surface // errors before returning the handle. let (open_tx, open_rx) = mpsc::channel::>(); let (cmd_tx, cmd_rx) = mpsc::channel::<(WriterCmd, mpsc::Sender)>(); let join = thread::spawn(move || { let file = ::gio::File::for_uri(&url); let stream = match file.replace( None, false, ::gio::FileCreateFlags::REPLACE_DESTINATION, ::gio::Cancellable::NONE, ) { Ok(s) => { if open_tx.send(Ok(())).is_err() { // Caller went away; clean up and exit. let _ = s.close(::gio::Cancellable::NONE); return; } s } Err(e) => { let _ = open_tx.send(Err(e.to_string())); return; } }; while let Ok((cmd, reply)) = cmd_rx.recv() { match cmd { WriterCmd::Write(buf) => { let res = match stream.write_all(&buf, ::gio::Cancellable::NONE) { Ok((written, None)) => Ok(written), Ok((_, Some(e))) => Err(e.to_string()), Err(e) => Err(e.to_string()), }; let _ = reply.send(res); } WriterCmd::Flush => { let res = stream .flush(::gio::Cancellable::NONE) .map(|_| 0usize) .map_err(|e| e.to_string()); let _ = reply.send(res); } WriterCmd::Close => { let res = stream .close(::gio::Cancellable::NONE) .map(|_| 0usize) .map_err(|e| e.to_string()); let _ = reply.send(res); return; } } } // Sender dropped without an explicit Close — best-effort close. let _ = stream.close(::gio::Cancellable::NONE); }); match open_rx.recv() { Ok(Ok(())) => Ok(GioWriteStream { tx: Some(cmd_tx), join: Some(join), }), Ok(Err(msg)) => { let _ = join.join(); Err(Error::Io(std::io::Error::other(msg))) } Err(_) => { let _ = join.join(); Err(Error::Io(std::io::Error::other( "gio writer thread exited before opening stream", ))) } } } fn dispatch(&self, cmd: WriterCmd) -> std::io::Result { let tx = self .tx .as_ref() .ok_or_else(|| std::io::Error::other("gio write stream already closed"))?; let (reply_tx, reply_rx) = mpsc::channel(); tx.send((cmd, reply_tx)) .map_err(|_| std::io::Error::other("gio writer thread exited"))?; match reply_rx.recv() { Ok(Ok(n)) => Ok(n), Ok(Err(msg)) => Err(std::io::Error::other(msg)), Err(_) => Err(std::io::Error::other("gio writer thread exited")), } } } impl std::io::Write for GioWriteStream { fn write(&mut self, buf: &[u8]) -> std::io::Result { // write_all on the worker drains the whole buffer or returns an error. self.dispatch(WriterCmd::Write(buf.to_vec()))?; Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { self.dispatch(WriterCmd::Flush).map(|_| ()) } } impl WriteStream for GioWriteStream { fn sync_data(&self) -> std::io::Result<()> { // gvfs has no fsync; OutputStream::flush is the strongest durability // primitive available, matching what the Python port did. self.dispatch(WriterCmd::Flush).map(|_| ()) } } impl Drop for GioWriteStream { fn drop(&mut self) { // Best-effort close. Errors here have nowhere to go. if let Some(tx) = self.tx.take() { let (reply_tx, reply_rx) = mpsc::channel(); if tx.send((WriterCmd::Close, reply_tx)).is_ok() { let _ = reply_rx.recv(); } } if let Some(join) = self.join.take() { let _ = join.join(); } } } /// gvfs offers no real lock primitive, matching the Python implementation /// which returned a no-op lock. We do the same. struct BogusLock; impl Lock for BogusLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { Ok(()) } } impl Transport for GioTransport { fn external_url(&self) -> Result { Ok(self.base.clone()) } fn can_roundtrip_unix_modebits(&self) -> bool { false } fn base(&self) -> Url { self.base.clone() } fn get(&self, relpath: &UrlFragment) -> Result> { let f = self.file_for(relpath)?; let input = f .read(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath)))?; let mut buf = Vec::new(); loop { let chunk = input .read_bytes(64 * 1024, ::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath)))?; if chunk.is_empty() { break; } buf.extend_from_slice(&chunk); } let _ = input.close(::gio::Cancellable::NONE); Ok(Box::new(GioReadStream(Cursor::new(buf)))) } fn has(&self, relpath: &UrlFragment) -> Result { let f = self.file_for(relpath)?; match f.query_info( "standard::type", FileQueryInfoFlags::NONE, ::gio::Cancellable::NONE, ) { Ok(info) => Ok(matches!( info.file_type(), ::gio::FileType::Regular | ::gio::FileType::Directory )), Err(e) if e.kind::() == Some(IOErrorEnum::NotFound) => Ok(false), Err(e) => Err(Self::translate(e, Some(relpath))), } } fn mkdir(&self, relpath: &UrlFragment, _permissions: Option) -> Result<()> { let f = self.file_for(relpath)?; f.make_directory(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath))) } fn stat(&self, relpath: &UrlFragment) -> Result { let f = self.file_for(relpath)?; let info = f .query_info( "standard::size,standard::type", FileQueryInfoFlags::NONE, ::gio::Cancellable::NONE, ) .map_err(|e| Self::translate(e, Some(relpath)))?; let kind = match info.file_type() { ::gio::FileType::Regular => FileKind::File, ::gio::FileType::Directory => FileKind::Dir, ::gio::FileType::SymbolicLink => FileKind::Symlink, _ => FileKind::Other, }; Ok(Stat { size: info.size().max(0) as usize, #[cfg(unix)] mode: match kind { FileKind::Dir => 0o040755, FileKind::Symlink => 0o120777, _ => 0o100644, }, kind, mtime: None, }) } fn clone(&self, offset: Option<&UrlFragment>) -> Result> { let new_backend = match offset { Some(o) if !o.is_empty() => { let base = Url::parse(&self.backend_url).map_err(Error::from)?; base.join(o).map_err(Error::from)?.to_string() } _ => self.backend_url.clone(), }; let new_base = format!("gio+{}", new_backend); Ok(Box::new(GioTransport::new(&new_base)?)) } fn abspath(&self, relpath: &UrlFragment) -> Result { let trimmed = if relpath == "." || relpath.is_empty() { "" } else { relpath }; self.base.join(trimmed).map_err(Error::from) } fn relpath(&self, abspath: &Url) -> Result { let base = self.base.as_str(); abspath .as_str() .strip_prefix(base) .map(|s| s.to_string()) .ok_or(Error::PathNotChild) } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn Read, _permissions: Option, ) -> Result { // Mirror Python: write to a temp sibling, then move-with-overwrite. let tmp_rel = format!("{}.tmp.{}", relpath, std::process::id()); let tmp_file = self.file_for(&tmp_rel)?; let dest_file = self.file_for(relpath)?; let mut buf = Vec::new(); f.read_to_end(&mut buf) .map_err(|e| Error::Io(std::io::Error::other(e.to_string())))?; let out = tmp_file .create(::gio::FileCreateFlags::NONE, ::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath)))?; // OutputStreamExtManual::write_all loops until the buffer drains. // Signature: Result<(written, Option), full_err>. match out.write_all(&buf, ::gio::Cancellable::NONE) { Ok((_, None)) => {} Ok((_, Some(e))) => return Err(Self::translate(e, Some(relpath))), Err(e) => return Err(Self::translate(e, Some(relpath))), } out.close(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath)))?; let move_result = tmp_file.move_( &dest_file, FileCopyFlags::OVERWRITE, ::gio::Cancellable::NONE, None, ); if let Err(e) = move_result { // Best-effort cleanup; ignore secondary errors. let _ = tmp_file.delete(::gio::Cancellable::NONE); return Err(Self::translate(e, Some(relpath))); } Ok(buf.len() as u64) } fn delete(&self, relpath: &UrlFragment) -> Result<()> { let f = self.file_for(relpath)?; f.delete(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath))) } fn rmdir(&self, relpath: &UrlFragment) -> Result<()> { let st = self.stat(relpath)?; if st.kind != FileKind::Dir { return Err(Error::NotADirectoryError(Some(relpath.to_string()))); } let f = self.file_for(relpath)?; f.delete(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath))) } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let from = self.file_for(rel_from)?; let to = self.file_for(rel_to)?; from.move_(&to, FileCopyFlags::NONE, ::gio::Cancellable::NONE, None) .map_err(|e| Self::translate(e, Some(rel_from))) } fn r#move(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let from = self.file_for(rel_from)?; let to = self.file_for(rel_to)?; from.move_( &to, FileCopyFlags::OVERWRITE, ::gio::Cancellable::NONE, None, ) .map_err(|e| Self::translate(e, Some(rel_from))) } fn copy(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let data = self.get_bytes(rel_from)?; let mut cur = Cursor::new(data); self.put_file(rel_to, &mut cur, None).map(|_| ()) } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn Read, _permissions: Option, ) -> Result { // Python notes that GIO's append_to truncates instead of appending, // so it implements a manual read+rewrite-via-tempfile. Mirror that. let mut existing = match self.get_bytes(relpath) { Ok(b) => b, Err(Error::NoSuchFile(_)) => Vec::new(), Err(e) => return Err(e), }; let original_len = existing.len() as u64; let mut to_append = Vec::new(); f.read_to_end(&mut to_append) .map_err(|e| Error::Io(std::io::Error::other(e.to_string())))?; existing.extend_from_slice(&to_append); let mut cur = Cursor::new(existing); self.put_file(relpath, &mut cur, None)?; Ok(original_len) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { let f = match self.file_for(relpath) { Ok(f) => f, Err(e) => return Box::new(std::iter::once(Err(e))), }; let enumerator = match f.enumerate_children( "standard::name", FileQueryInfoFlags::NONE, ::gio::Cancellable::NONE, ) { Ok(e) => e, Err(e) => return Box::new(std::iter::once(Err(Self::translate(e, Some(relpath))))), }; let mut entries: Vec> = Vec::new(); loop { match enumerator.next_file(::gio::Cancellable::NONE) { Ok(Some(info)) => { let name = info.name(); let name_str = name.to_string_lossy(); entries.push(Ok(escape(name_str.as_bytes(), None))); } Ok(None) => break, Err(e) => { entries.push(Err(Self::translate(e, Some(relpath)))); break; } } } let _ = enumerator.close(::gio::Cancellable::NONE); Box::new(entries.into_iter()) } fn iter_files_recursive(&self) -> Box>> { let mut queue: Vec = Vec::new(); for entry in self.list_dir(".") { match entry { Ok(name) => queue.push(name), Err(e) => return Box::new(std::iter::once(Err(e))), } } let mut results: Vec> = Vec::new(); while let Some(rel) = queue.pop() { match self.stat(&rel) { Ok(st) if st.kind == FileKind::Dir => { for child in self.list_dir(&rel) { match child { Ok(name) => queue.push(format!("{}/{}", rel, name)), Err(e) => { results.push(Err(e)); break; } } } } Ok(_) => results.push(Ok(rel)), Err(e) => results.push(Err(e)), } } Box::new(results.into_iter()) } fn lock_read(&self, _relpath: &UrlFragment) -> Result> { Ok(Box::new(BogusLock)) } fn lock_write(&self, _relpath: &UrlFragment) -> Result> { Ok(Box::new(BogusLock)) } fn local_abspath(&self, relpath: &UrlFragment) -> Result { Err(Error::NotLocalUrl(format!("{}{}", self.base, relpath))) } fn get_smart_medium(&self) -> Result> { Err(Error::NoSmartMedium) } fn listable(&self) -> bool { true } fn set_segment_parameter(&mut self, _key: &str, _value: Option<&str>) -> Result<()> { Err(Error::TransportNotPossible) } fn get_segment_parameters(&self) -> Result> { Ok(HashMap::new()) } fn readlink(&self, _relpath: &UrlFragment) -> Result { // gvfs symlinks are exposed by query_info but resolving them // requires standard::symlink-target. The Python port did not // implement readlink either; keep parity. Err(Error::TransportNotPossible) } fn hardlink(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(Error::TransportNotPossible) } fn symlink(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(Error::TransportNotPossible) } fn delete_tree(&self, relpath: &UrlFragment) -> Result<()> { let st = self.stat(relpath)?; if st.kind != FileKind::Dir { return Err(Error::NotADirectoryError(Some(relpath.to_string()))); } // Depth-first removal: enumerate entries, recurse into directories, // delete files, then delete the now-empty directory. let f = self.file_for(relpath)?; let enumerator = f .enumerate_children( "standard::name,standard::type", FileQueryInfoFlags::NOFOLLOW_SYMLINKS, ::gio::Cancellable::NONE, ) .map_err(|e| Self::translate(e, Some(relpath)))?; loop { match enumerator.next_file(::gio::Cancellable::NONE) { Ok(Some(info)) => { let name = info.name(); let name_str = name.to_string_lossy(); let child_rel = format!("{}/{}", relpath.trim_end_matches('/'), name_str); match info.file_type() { ::gio::FileType::Directory => self.delete_tree(&child_rel)?, _ => { let child = self.file_for(&child_rel)?; child .delete(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(&child_rel)))?; } } } Ok(None) => break, Err(e) => return Err(Self::translate(e, Some(relpath))), } } let _ = enumerator.close(::gio::Cancellable::NONE); f.delete(::gio::Cancellable::NONE) .map_err(|e| Self::translate(e, Some(relpath))) } fn open_write_stream( &self, relpath: &UrlFragment, _permissions: Option, ) -> Result> { // gio::FileOutputStream is !Send. We dedicate one worker thread per // open stream — the worker owns the underlying handle and we drive // it via a synchronous command channel. let url = self.child_url(relpath)?; let stream = GioWriteStream::spawn(url)?; Ok(Box::new(stream)) } } // `gio::File` and friends are `!Send + !Sync`. Our struct only stores // owned `String`/`Url`, which are Send+Sync, and `GioWriteStream` keeps // its !Send `gio::FileOutputStream` pinned to a worker thread. #[cfg(test)] mod tests { use super::*; use tempfile::TempDir; fn temp_transport() -> (TempDir, GioTransport) { let dir = TempDir::new().unwrap(); // gio::File::for_uri wants file:///abs/path — build it via Url so // path escaping is handled for us. let file_url = url::Url::from_directory_path(dir.path()).unwrap(); let base = format!("gio+{}", file_url.as_str()); let t = GioTransport::new(&base).unwrap(); (dir, t) } #[test] fn rejects_unknown_scheme() { match GioTransport::new("gio+nope:///x/") { Err(Error::UrlError(_)) => {} other => panic!("expected UrlError, got {:?}", other), } } #[test] fn requires_gio_prefix() { match GioTransport::new("file:///tmp/") { Err(Error::NotLocalUrl(_)) => {} other => panic!("expected NotLocalUrl, got {:?}", other), } } #[test] fn put_get_has_round_trip() { let (_dir, t) = temp_transport(); assert!(!t.has("hello").unwrap()); t.put_bytes("hello", b"world", None).unwrap(); assert!(t.has("hello").unwrap()); assert_eq!(t.get_bytes("hello").unwrap(), b"world"); } #[test] fn mkdir_stat_list_round_trip() { let (_dir, t) = temp_transport(); t.mkdir("d", None).unwrap(); t.put_bytes("d/a", b"1", None).unwrap(); t.put_bytes("d/b", b"22", None).unwrap(); let mut entries: Vec = t.list_dir("d").filter_map(|r| r.ok()).collect(); entries.sort(); assert_eq!(entries, vec!["a".to_string(), "b".to_string()]); assert_eq!(t.stat("d").unwrap().kind, FileKind::Dir); } #[test] fn rename_and_delete() { let (_dir, t) = temp_transport(); t.put_bytes("a", b"hi", None).unwrap(); t.rename("a", "b").unwrap(); assert!(!t.has("a").unwrap()); assert_eq!(t.get_bytes("b").unwrap(), b"hi"); t.delete("b").unwrap(); assert!(!t.has("b").unwrap()); } #[test] fn append_extends_file() { let (_dir, t) = temp_transport(); t.put_bytes("f", b"abc", None).unwrap(); let mut more = Cursor::new(b"DEF".to_vec()); let offset = t.append_file("f", &mut more, None).unwrap(); assert_eq!(offset, 3); assert_eq!(t.get_bytes("f").unwrap(), b"abcDEF"); } #[test] fn missing_file_get_returns_no_such_file() { let (_dir, t) = temp_transport(); match t.get_bytes("nope") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn open_write_stream_round_trip() { use std::io::Write; let (_dir, t) = temp_transport(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"hello ").unwrap(); stream.write_all(b"world").unwrap(); stream.flush().unwrap(); drop(stream); assert_eq!(t.get_bytes("w").unwrap(), b"hello world"); } #[test] fn open_write_stream_visible_after_flush() { // After explicit flush, a concurrent read on the same path must see // the buffered writes — this is what the per_transport // test_get_with_open_write_stream_sees_all_content scenario asserts. use std::io::Write; let (_dir, t) = temp_transport(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"bcd").unwrap(); stream.flush().unwrap(); assert_eq!(t.get_bytes("w").unwrap(), b"bcd"); drop(stream); } #[test] fn open_write_stream_overwrites_existing() { use std::io::Write; let (_dir, t) = temp_transport(); t.put_bytes("w", b"old contents", None).unwrap(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"new").unwrap(); drop(stream); assert_eq!(t.get_bytes("w").unwrap(), b"new"); } #[test] fn delete_tree_removes_nested() { let (_dir, t) = temp_transport(); t.mkdir("d", None).unwrap(); t.put_bytes("d/a", b"1", None).unwrap(); t.mkdir("d/sub", None).unwrap(); t.put_bytes("d/sub/b", b"2", None).unwrap(); t.delete_tree("d").unwrap(); assert!(!t.has("d").unwrap()); } #[test] fn delete_tree_rejects_non_directory() { let (_dir, t) = temp_transport(); t.put_bytes("f", b"x", None).unwrap(); match t.delete_tree("f") { Err(Error::NotADirectoryError(_)) => {} other => panic!("expected NotADirectoryError, got {:?}", other), } } #[test] fn iter_files_recursive_walks() { let (_dir, t) = temp_transport(); t.mkdir("d", None).unwrap(); t.put_bytes("d/a", b"1", None).unwrap(); t.mkdir("d/sub", None).unwrap(); t.put_bytes("d/sub/b", b"2", None).unwrap(); let mut files: Vec = t.iter_files_recursive().filter_map(|r| r.ok()).collect(); files.sort(); assert_eq!(files, vec!["d/a".to_string(), "d/sub/b".to_string()]); } } dromedary-0.1.1/src/lib.rs000064400000000000000000000422671046102023000135270ustar 00000000000000use crate::lock::{Lock, LockError}; use std::collections::HashMap; use std::fs::{Metadata, Permissions}; use std::io::{Read, Seek}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::time::UNIX_EPOCH; use url::Url; #[derive(Debug)] pub enum Error { InProcessTransport, NoSmartMedium, NotLocalUrl(String), NoSuchFile(Option), FileExists(Option), TransportNotPossible, UrlError(url::ParseError), UrlutilsError(crate::urlutils::Error), PermissionDenied(Option), Io(std::io::Error), PathNotChild, UnexpectedEof, ShortReadvError(String, u64, u64, u64), LockContention(std::path::PathBuf), LockFailed(std::path::PathBuf, String), IsADirectoryError(Option), NotADirectoryError(Option), DirectoryNotEmptyError(Option), ResourceBusy(Option), } pub type Result = std::result::Result; pub type UrlFragment = str; pub fn map_io_err_to_transport_err(err: std::io::Error, path: Option<&str>) -> Error { match err.kind() { std::io::ErrorKind::NotFound => Error::NoSuchFile(path.map(|p| p.to_string())), std::io::ErrorKind::AlreadyExists => Error::FileExists(path.map(|p| p.to_string())), std::io::ErrorKind::PermissionDenied => { Error::PermissionDenied(path.map(|p| p.to_string())) } // use of unstable library feature 'io_error_more' // https://github.com/rust-lang/rust/issues/86442 // // std::io::ErrorKind::NotADirectoryError => Error::NotADirectoryError(None), // std::io::ErrorKind::IsADirectoryError => Error::IsADirectoryError(None), _ => match err.raw_os_error() { Some(e) if e == libc::ENOTDIR => Error::NotADirectoryError(path.map(|p| p.to_string())), Some(e) if e == libc::EISDIR => Error::IsADirectoryError(path.map(|p| p.to_string())), Some(e) if e == libc::ENOTEMPTY => { Error::DirectoryNotEmptyError(path.map(|p| p.to_string())) } _ => Error::Io(err), }, } } impl From for Error { fn from(err: url::ParseError) -> Self { Error::UrlError(err) } } impl From for Error { fn from(err: crate::urlutils::Error) -> Self { Error::UrlutilsError(err) } } /// Coarse file kind. Mirrors `std::fs::FileType` but is cross-platform and /// sidesteps the Unix-only mode-bit parsing the old implementation relied on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FileKind { File, Dir, Symlink, Other, } pub struct Stat { pub size: usize, /// Unix permission bits. Not present on Windows — see /// `memory/project_windows_port.md` for the design rationale. #[cfg(unix)] pub mode: u32, pub kind: FileKind, pub mtime: Option, } impl From for Stat { fn from(metadata: Metadata) -> Self { let ft = metadata.file_type(); let kind = if ft.is_dir() { FileKind::Dir } else if ft.is_file() { FileKind::File } else if ft.is_symlink() { FileKind::Symlink } else { FileKind::Other }; Stat { size: metadata.len() as usize, #[cfg(unix)] mode: metadata.permissions().mode(), kind, mtime: metadata.modified().map_or(None, |t| { Some(t.duration_since(UNIX_EPOCH).unwrap().as_secs_f64()) }), } } } impl Stat { pub fn is_dir(&self) -> bool { self.kind == FileKind::Dir } pub fn is_file(&self) -> bool { self.kind == FileKind::File } } pub trait WriteStream: std::io::Write { fn sync_data(&self) -> std::io::Result<()>; } pub trait ReadStream: Read + Seek {} pub trait Transport: std::fmt::Debug + 'static + Send + Sync { /// Return a URL for self that can be given to an external process. /// /// There is no guarantee that the URL can be accessed from a different /// machine - e.g. file:/// urls are only usable on the local machine, /// sftp:/// urls when the server is only bound to localhost are only /// usable from localhost etc. /// /// NOTE: This method may remove security wrappers (e.g. on chroot /// transports) and thus should *only* be used when the result will not /// be used to obtain a new transport within breezy. Ideally chroot /// transports would know enough to cause the external url to be the exact /// one used that caused the chrooting in the first place, but that is not /// currently the case. /// /// Returns: A URL that can be given to another process. /// Raises:InProcessTransport: If the transport is one that cannot be /// accessed out of the current process (e.g. a MemoryTransport) /// then InProcessTransport is raised. fn external_url(&self) -> Result; fn can_roundtrip_unix_modebits(&self) -> bool; fn get_bytes(&self, relpath: &UrlFragment) -> Result> { let mut file = self.get(relpath)?; let mut result = Vec::new(); file.read_to_end(&mut result) .map_err(|err| map_io_err_to_transport_err(err, Some(relpath)))?; Ok(result) } fn get(&self, relpath: &UrlFragment) -> Result>; fn base(&self) -> Url; /// Ensure that the directory this transport references exists. /// /// This will create a directory if it doesn't exist. /// Returns: True if the directory was created, False otherwise. fn ensure_base(&self, permissions: Option) -> Result { if let Err(err) = self.mkdir(".", permissions) { match err { Error::FileExists(_) => Ok(false), Error::PermissionDenied(_) => Ok(false), Error::TransportNotPossible => { if self.has(".")? { Ok(false) } else { Err(err) } } _ => Err(err), } } else { Ok(true) } } fn create_prefix(&self, permissions: Option) -> Result<()> { let mut cur_transport = self.clone(None)?; let mut needed = vec![]; loop { match cur_transport.mkdir(".", permissions.clone()) { Err(Error::NoSuchFile(_)) => { let new_transport = Transport::clone(cur_transport.as_ref(), Some(".."))?; assert_ne!( new_transport.base(), cur_transport.base(), "Failed to create path prefix for {}", cur_transport.base() ); needed.push(cur_transport); cur_transport = new_transport; } Err(Error::FileExists(_)) | Ok(()) => { break; } Err(err) => { return Err(err); } } } while let Some(transport) = needed.pop() { transport.ensure_base(permissions.clone())?; } Ok(()) } fn has(&self, relpath: &UrlFragment) -> Result; fn has_any(&self, relpaths: &[&UrlFragment]) -> Result { for relpath in relpaths { if self.has(relpath)? { return Ok(true); } } Ok(false) } fn mkdir(&self, relpath: &UrlFragment, permissions: Option) -> Result<()>; fn stat(&self, relpath: &UrlFragment) -> Result; fn clone(&self, offset: Option<&UrlFragment>) -> Result>; fn abspath(&self, relpath: &UrlFragment) -> Result; fn relpath(&self, abspath: &Url) -> Result; fn put_file( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, ) -> Result; fn put_bytes( &self, relpath: &UrlFragment, data: &[u8], permissions: Option, ) -> Result<()> { let mut f = std::io::Cursor::new(data); self.put_file(relpath, &mut f, permissions)?; Ok(()) } fn put_file_non_atomic( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, create_parent_dir: Option, dir_permissions: Option, ) -> Result<()> { match self.put_file(relpath, f, permissions.clone()) { Ok(_) => Ok(()), Err(Error::NoSuchFile(filename)) => { if create_parent_dir.unwrap_or(false) { if let Some(parent) = relpath.rsplit_once('/').map(|x| x.0) { self.mkdir(parent, dir_permissions)?; self.put_file(relpath, f, permissions)?; Ok(()) } else { Err(Error::NoSuchFile(filename)) } } else { Err(Error::NoSuchFile(filename)) } } Err(err) => Err(err), } } fn put_bytes_non_atomic( &self, relpath: &UrlFragment, data: &[u8], permissions: Option, create_parent_dir: Option, dir_permissions: Option, ) -> Result<()> { let mut f = std::io::Cursor::new(data); self.put_file_non_atomic( relpath, &mut f, permissions, create_parent_dir, dir_permissions, ) } fn delete(&self, relpath: &UrlFragment) -> Result<()>; fn rmdir(&self, relpath: &UrlFragment) -> Result<()>; fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()>; fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> Result<()>; fn get_segment_parameters(&self) -> Result>; /// Return the recommended page size for this transport. /// /// This is potentially different for every path in a given namespace. /// For example, local transports might use an operating system call to /// get the block size for a given path, which can vary due to mount /// points. /// /// Returns: The page size in bytes. fn recommended_page_size(&self) -> usize { 4 * 1024 } fn is_readonly(&self) -> bool { false } fn readv<'a>( &self, relpath: &'a UrlFragment, offsets: Vec<(u64, usize)>, adjust_for_latency: bool, upper_limit: Option, ) -> Box)>> + Send + 'a> { let offsets = if adjust_for_latency { crate::readv::sort_expand_and_combine( offsets, upper_limit, self.recommended_page_size(), ) } else { offsets }; let buf = match self.get_bytes(relpath) { Err(err) => return Box::new(std::iter::once(Err(err))), Ok(file) => file, }; let mut file = std::io::Cursor::new(buf); Box::new( offsets .into_iter() .map(move |(offset, length)| -> Result<(u64, Vec)> { let mut buf = vec![0; length]; match file.seek(std::io::SeekFrom::Start(offset)) { Ok(_) => {} Err(err) => match err.kind() { std::io::ErrorKind::UnexpectedEof => { return Err(Error::ShortReadvError( relpath.to_owned(), offset, length as u64, file.position() - offset, )) } _ => return Err(map_io_err_to_transport_err(err, Some(relpath))), }, } match file.read_exact(&mut buf) { Ok(_) => Ok((offset, buf)), Err(err) => match err.kind() { std::io::ErrorKind::UnexpectedEof => Err(Error::ShortReadvError( relpath.to_owned(), offset, length as u64, file.position().saturating_sub(offset), )), _ => Err(map_io_err_to_transport_err(err, Some(relpath))), }, } }), ) } fn append_bytes( &self, relpath: &UrlFragment, data: &[u8], permissions: Option, ) -> Result { let mut f = std::io::Cursor::new(data); self.append_file(relpath, &mut f, permissions) } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn std::io::Read, permissions: Option, ) -> Result; fn readlink(&self, relpath: &UrlFragment) -> Result; fn hardlink(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()>; fn symlink(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()>; fn iter_files_recursive(&self) -> Box>>; fn open_write_stream( &self, relpath: &UrlFragment, permissions: Option, ) -> Result>; fn delete_tree(&self, relpath: &UrlFragment) -> Result<()>; fn r#move(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()>; fn copy_tree(&self, from_relpath: &UrlFragment, to_relpath: &UrlFragment) -> Result<()> { let source = self.clone(Some(from_relpath))?; let target = self.clone(Some(to_relpath))?; // create target directory with the same rwx bits as source // use umask to ensure bits other than rwx are ignored let stat = self.stat(from_relpath)?; #[cfg(unix)] let perms = Some(Permissions::from_mode(stat.mode)); #[cfg(not(unix))] let perms: Option = { let _ = stat; None }; target.mkdir(".", perms)?; source.copy_tree_to_transport(target.as_ref())?; Ok(()) } fn copy_tree_to_transport(&self, to_transport: &dyn Transport) -> Result<()> { let mut files = Vec::new(); let mut directories = vec![".".to_string()]; while let Some(dir) = directories.pop() { if dir != "." { to_transport.mkdir(dir.as_str(), None)?; } for entry in self.list_dir(dir.as_str()) { let entry = entry?; let full_path = format!("{}/{}", dir, entry); let stat = self.stat(&full_path)?; if stat.is_dir() { directories.push(full_path); } else { files.push(full_path); } } } self.copy_to( files .iter() .map(|x| x.as_str()) .collect::>() .as_slice(), to_transport, None, )?; Ok(()) } fn copy_to( &self, relpaths: &[&UrlFragment], to_transport: &dyn Transport, permissions: Option, ) -> Result { copy_to(self, to_transport, relpaths, permissions) } fn list_dir(&self, relpath: &UrlFragment) -> Box>>; fn listable(&self) -> bool { true } fn lock_read(&self, relpath: &UrlFragment) -> Result>; fn lock_write(&self, relpath: &UrlFragment) -> Result>; fn local_abspath(&self, relpath: &UrlFragment) -> Result; fn get_smart_medium(&self) -> Result>; fn copy(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()>; } pub fn copy_to( from_transport: &T, to_transport: &dyn Transport, relpaths: &[&UrlFragment], permissions: Option, ) -> Result { let mut count = 0; relpaths.iter().try_for_each(|relpath| -> Result<()> { let mut src = from_transport.get(relpath)?; let mut target = to_transport.open_write_stream(relpath, permissions.clone())?; std::io::copy(&mut src, &mut target) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; count += 1; Ok(()) })?; Ok(count) } pub trait SmartMedium {} pub mod local; pub mod brokenrename; pub mod chroot; pub mod decorator; pub mod fakenfs; pub mod fakevfat; #[cfg(feature = "gio")] pub mod gio; pub mod memory; pub mod pathfilter; pub mod readonly; pub mod unlistable; pub mod osutils; #[cfg(feature = "pyo3")] pub mod pyo3; pub mod readv; #[cfg(unix)] #[path = "fcntl-locks.rs"] pub mod filelock; #[cfg(target_os = "windows")] #[path = "win32-locks.rs"] pub mod filelock; pub mod lock; pub mod urlutils; dromedary-0.1.1/src/local.rs000064400000000000000000000422041046102023000140420ustar 00000000000000use crate::lock::LockError; use crate::urlutils::{escape, unescape}; use crate::{ map_io_err_to_transport_err, Error, Lock, ReadStream, Result, SmartMedium, Stat, Transport, UrlFragment, WriteStream, }; use std::collections::HashMap; use std::convert::TryFrom; use std::fs::File; use std::fs::Permissions; use std::io::{Read, Seek}; use std::path::{Path, PathBuf}; use url::Url; use walkdir; pub struct LocalTransport { base: Url, path: PathBuf, } impl TryFrom<&Path> for LocalTransport { type Error = Error; fn try_from(path: &Path) -> Result { let url = crate::urlutils::local_path_to_url(path).map_err(|e| { map_io_err_to_transport_err(e, Some(path.to_path_buf().to_str().unwrap())) })?; LocalTransport::new(&url) } } impl TryFrom for LocalTransport { type Error = Error; fn try_from(url: Url) -> Result { LocalTransport::new(url.as_str()) } } impl Clone for LocalTransport { fn clone(&self) -> Self { LocalTransport { path: self.path.clone(), base: self.base.clone(), } } } impl WriteStream for File { fn sync_data(&self) -> std::io::Result<()> { self.sync_data() } } impl ReadStream for File {} impl LocalTransport { pub fn new(base: &str) -> Result { let base = if base.ends_with('/') { base.to_string() } else { format!("{}/", base) }; let mut path = crate::urlutils::local_path_from_url(&base)?; if !path.to_string_lossy().ends_with('/') { path.push("") } let base = Url::parse(&base)?; Ok(LocalTransport { base, path }) } fn _abspath(&self, relative_reference: &str) -> Result { if relative_reference == "." || relative_reference.is_empty() { Ok(self.path.clone()) } else { let mut ret = self.path.clone(); let extra = crate::urlutils::unescape(relative_reference)?; let extra = extra.trim_start_matches('/'); ret.push(extra); Ok(ret) } } } impl std::fmt::Debug for LocalTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "LocalTransport({})", self.base) } } fn lock_err_to_transport_err(e: LockError) -> Error { match e { LockError::Contention(p) => Error::LockContention(p), LockError::IoError(e) => Error::Io(e), LockError::Failed(p, w) => Error::LockFailed(p, w), } } impl Transport for LocalTransport { fn external_url(&self) -> Result { Ok(self.base.clone()) } fn base(&self) -> Url { self.base.clone() } fn local_abspath(&self, relpath: &UrlFragment) -> Result { let absurl = self.abspath(relpath)?; crate::urlutils::local_path_from_url(absurl.as_str()).map_err(Error::from) } fn can_roundtrip_unix_modebits(&self) -> bool { #[cfg(unix)] return true; #[cfg(not(unix))] return false; } fn get(&self, relpath: &UrlFragment) -> Result> { let path = self._abspath(relpath)?; let f = std::fs::File::open(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; Ok(Box::new(f)) } fn mkdir(&self, relpath: &UrlFragment, permissions: Option) -> Result<()> { let path = self._abspath(relpath)?; std::fs::create_dir(&path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; if let Some(permissions) = permissions { std::fs::set_permissions(&path, permissions) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; } Ok(()) } fn has(&self, path: &UrlFragment) -> Result { let path = self._abspath(path)?; Ok(path.exists()) } fn stat(&self, relpath: &UrlFragment) -> Result { use std::ffi::OsStr; let path = self._abspath(relpath)?; // Strip trailing slashes, so we can properly stat broken symlinks. // We work on the encoded bytes of the OsStr so this stays // WTF-8-safe on Windows and UTF-8-safe on Unix. let path = { let b = path.as_path().as_os_str().as_encoded_bytes(); if let Some(stripped) = b.strip_suffix(b"/") { // SAFETY: we only stripped an ASCII byte; the rest of the // encoding is intact. PathBuf::from(unsafe { OsStr::from_encoded_bytes_unchecked(stripped) }) } else { path } }; Ok(Stat::from(std::fs::symlink_metadata(path).map_err( |e| map_io_err_to_transport_err(e, Some(relpath)), )?)) } fn clone(&self, offset: Option<&UrlFragment>) -> Result> { let new_base = match offset { Some(offset) => self.abspath(offset)?, None => self.base.clone(), }; Ok(Box::new(LocalTransport::new(new_base.as_str())?)) } fn abspath(&self, relpath: &UrlFragment) -> Result { let path = self.path.join(unescape(relpath)?); let path = crate::osutils::path::normpath(path); let url_str = crate::urlutils::local_path_to_url(path.as_path()) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; Ok(Url::parse(&url_str).unwrap()) } fn relpath(&self, abspath: &Url) -> Result { let relpath = crate::urlutils::file_relpath(self.base.as_str(), abspath.as_str()) .map_err(Error::from)?; Ok(relpath) } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, ) -> Result { let path = self._abspath(relpath)?; let mut tmpfile = tempfile::Builder::new() .tempfile_in(path.parent().unwrap()) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; let n = std::io::copy(f, &mut tmpfile) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; let f = tmpfile .persist(&path) .map_err(|e| map_io_err_to_transport_err(e.error, Some(relpath)))?; if let Some(permissions) = permissions { f.set_permissions(permissions) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; } Ok(n) } fn delete(&self, relpath: &UrlFragment) -> Result<()> { let path = self._abspath(relpath)?; std::fs::remove_file(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath))) } fn rmdir(&self, relpath: &UrlFragment) -> Result<()> { let path = self._abspath(relpath)?; std::fs::remove_dir(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath))) } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let abs_from = self._abspath(rel_from)?; let abs_to = self._abspath(rel_to)?; std::fs::rename(abs_from, abs_to) .map_err(|e| map_io_err_to_transport_err(e, Some(rel_from))) } fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> Result<()> { let (raw, mut params) = crate::urlutils::split_segment_parameters(self.base.as_str())?; if let Some(value) = value { params.insert(key, value); } else { params.remove(key); } self.base = Url::parse(&crate::urlutils::join_segment_parameters(raw, ¶ms)?)?; Ok(()) } fn get_segment_parameters(&self) -> Result> { let (_, params) = crate::urlutils::split_segment_parameters(self.base.as_str())?; Ok(params .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect()) } #[cfg(unix)] fn readv<'a>( &self, path: &'a UrlFragment, offsets: Vec<(u64, usize)>, adjust_for_latency: bool, upper_limit: Option, ) -> Box)>> + Send + 'a> { let offsets = if adjust_for_latency { crate::readv::sort_expand_and_combine( offsets, upper_limit, self.recommended_page_size(), ) } else { offsets }; use nix::libc::off_t; use nix::sys::uio::pread; let abspath = match self._abspath(path) { Ok(p) => p, Err(err) => return Box::new(std::iter::once(Err(err))), }; let file = match std::fs::File::open(abspath) { Ok(f) => f, Err(err) => { return Box::new(std::iter::once(Err(map_io_err_to_transport_err( err, Some(path), )))) } }; Box::new( offsets .into_iter() .map(move |(offset, len)| -> Result<(u64, Vec)> { let mut buf = vec![0; len]; match pread(&file, &mut buf[..], offset as off_t) { Ok(n) if n == len => Ok((offset, buf)), Ok(n) => Err(Error::ShortReadvError( path.to_owned(), offset, len as u64, n as u64, )), Err(e) => Err(map_io_err_to_transport_err( std::io::Error::from_raw_os_error(e as i32), Some(path), )), } }), ) } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn std::io::Read, permissions: Option, ) -> Result { let path = self._abspath(relpath)?; let mut file = std::fs::OpenOptions::new() .append(true) .create(true) .open(path) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; if let Some(permissions) = permissions { file.set_permissions(permissions) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; } let pos = file .seek(std::io::SeekFrom::End(0)) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; std::io::copy(f, &mut file).map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; Ok(pos) } fn readlink(&self, relpath: &UrlFragment) -> Result { let path = self._abspath(relpath)?; let target = std::fs::read_link(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; Ok(escape(target.as_os_str().as_encoded_bytes(), None)) } fn hardlink(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let from = self._abspath(rel_from)?; let to = self._abspath(rel_to)?; std::fs::hard_link(from, to).map_err(|e| map_io_err_to_transport_err(e, Some(rel_from))) } fn symlink(&self, source: &UrlFragment, link_name: &UrlFragment) -> Result<()> { let abs_to = self.abspath(link_name)?; let abs_link_dirpath = crate::urlutils::dirname(abs_to.as_str(), true); let source_rel = crate::urlutils::file_relpath( abs_link_dirpath.as_str(), self.abspath(source)?.as_str(), )?; #[cfg(unix)] { std::os::unix::fs::symlink(source_rel, self._abspath(link_name)?) .map_err(|e| map_io_err_to_transport_err(e, Some(link_name))) } #[cfg(windows)] { // On Windows `symlink_file` / `symlink_dir` require distinguishing // file vs directory targets and typically Administrator privilege // or Developer Mode. Emulate the common case (file symlink). // TODO(windows): pick `symlink_dir` when the source is a directory. let _ = source_rel; let _ = link_name; Err(Error::Io(std::io::Error::new( std::io::ErrorKind::Unsupported, "symlink is not implemented on Windows yet", ))) } } fn iter_files_recursive(&self) -> Box>> { let wd = walkdir::WalkDir::new(&self.path); fn walkdir_err(e: walkdir::Error) -> Error { let ioerr: std::io::Error = e.into(); map_io_err_to_transport_err(ioerr, None) } let base = self.path.clone(); Box::new(wd.into_iter().filter_map(move |e| match e { Ok(e) => { if !e.file_type().is_dir() { Some(Ok(escape( e.path() .strip_prefix(base.as_path()) .unwrap() .as_os_str() .as_encoded_bytes(), None, ))) } else { None } } Err(e) => Some(Err(walkdir_err(e))), })) } fn open_write_stream( &self, relpath: &UrlFragment, permissions: Option, ) -> Result> { let path = self._abspath(relpath)?; let file = File::create(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; file.set_len(0) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; if let Some(permissions) = permissions { file.set_permissions(permissions) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; } Ok(Box::new(file)) } fn delete_tree(&self, relpath: &UrlFragment) -> Result<()> { let path = self._abspath(relpath)?; std::fs::remove_dir_all(path).map_err(|e| map_io_err_to_transport_err(e, Some(relpath))) } fn r#move(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let from = self._abspath(rel_from)?; let to = self._abspath(rel_to)?; // TODO(jelmer): Should remove destination if necessary std::fs::rename(from, to).map_err(|e| map_io_err_to_transport_err(e, Some(rel_from))) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { let path = match self._abspath(relpath) { Ok(p) => p, Err(err) => return Box::new(std::iter::once(Err(err))), }; let entries = match std::fs::read_dir(path) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath))) { Ok(e) => e, Err(err) => return Box::new(std::iter::once(Err(err))), }; Box::new( entries .map(|entry| entry.map_err(|e| map_io_err_to_transport_err(e, None))) .map(|entry| { entry .map(|entry| escape(entry.file_name().as_os_str().as_encoded_bytes(), None)) }), ) } fn listable(&self) -> bool { true } fn get_smart_medium(&self) -> Result> { Err(Error::NoSmartMedium) } fn lock_read(&self, relpath: &UrlFragment) -> Result> { let path = self._abspath(relpath)?; let lock = crate::filelock::ReadLock::new(path.as_path(), false) .map_err(lock_err_to_transport_err)?; Ok(Box::new(lock)) } fn lock_write(&self, relpath: &UrlFragment) -> Result> { let path = self._abspath(relpath)?; let lock = crate::filelock::WriteLock::new(path.as_path(), false) .map_err(lock_err_to_transport_err)?; Ok(Box::new(lock)) } fn copy_to( &self, relpaths: &[&UrlFragment], target: &dyn Transport, permissions: Option, ) -> Result { if relpaths.is_empty() { return Ok(0); } match target.local_abspath(relpaths[0]) { // Fall back to default Err(Error::NotLocalUrl(_)) => { return super::copy_to(self, target, relpaths, permissions) } Err(e) => return Err(e), _ => {} } let mut count = 0; relpaths.iter().try_for_each(|relpath| { let path = self._abspath(relpath)?; let target_path = target.local_abspath(relpath)?; std::fs::copy(path, &target_path) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; if let Some(permissions) = permissions.clone() { std::fs::set_permissions(target_path, permissions) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; } count += 1; Ok::<(), Error>(()) })?; Ok(count) } fn copy(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { std::fs::copy( self._abspath(rel_from)?.as_path(), self._abspath(rel_to)?.as_path(), ) .map_err(|e| map_io_err_to_transport_err(e, Some(rel_from)))?; Ok(()) } } dromedary-0.1.1/src/lock.rs000064400000000000000000000012321046102023000136740ustar 00000000000000pub trait Lock { fn unlock(&mut self) -> std::result::Result<(), LockError>; } pub enum LockError { Contention(std::path::PathBuf), Failed(std::path::PathBuf, String), IoError(std::io::Error), } pub type LockResult = std::result::Result; impl From for LockError { fn from(err: std::io::Error) -> LockError { LockError::IoError(err) } } pub struct BogusLock; impl Lock for BogusLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { Ok(()) } } pub trait FileLock { fn file(&self) -> std::io::Result>; fn path(&self) -> &std::path::Path; } dromedary-0.1.1/src/memory.rs000064400000000000000000000705501046102023000142650ustar 00000000000000//! In-memory Transport implementation, ported from dromedary/memory.py. //! //! Storage is shared across clones via `Arc>`, matching //! the Python semantics where `clone()` passes the same dict references. use crate::lock::{Lock, LockError}; use crate::urlutils::{escape, unescape}; use crate::{ map_io_err_to_transport_err, Error, FileKind, ReadStream, Result, SmartMedium, Stat, Transport, UrlFragment, WriteStream, }; use std::collections::HashMap; use std::fs::Permissions; use std::io::{Cursor, Read, Write}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::sync::{Arc, Mutex}; use url::Url; /// Raw mode bits stored alongside each entry. On Unix we round-trip the full /// `Permissions` value; on Windows the concept is largely meaningless so we /// simply track the u32 that the caller supplied (if any), same as the /// Python implementation. type Mode = Option; fn perms_to_mode(p: Option) -> Mode { #[cfg(unix)] { p.map(|p| p.mode()) } #[cfg(not(unix))] { let _ = p; None } } #[derive(Default)] pub struct MemoryStore { files: HashMap, Mode)>, dirs: HashMap, symlinks: HashMap>, locks: HashMap, } impl MemoryStore { fn new() -> Self { let mut dirs = HashMap::new(); dirs.insert("/".to_string(), None); Self { files: HashMap::new(), dirs, symlinks: HashMap::new(), locks: HashMap::new(), } } } pub struct MemoryTransport { base: Url, scheme: String, cwd: String, store: Arc>, } impl std::fmt::Debug for MemoryTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "MemoryTransport({})", self.base) } } impl MemoryTransport { pub fn new(url: &str) -> Result { let mut url = url.to_string(); if url.is_empty() { url = "memory:///".to_string(); } if !url.ends_with('/') { url.push('/'); } let split = url .find(':') .ok_or_else(|| Error::NotLocalUrl(url.clone()))? + 3; if split > url.len() { return Err(Error::NotLocalUrl(url)); } let scheme = url[..split].to_string(); let cwd = url[split..].to_string(); let parsed = Url::parse(&url).map_err(Error::from)?; Ok(Self { base: parsed, scheme, cwd, store: Arc::new(Mutex::new(MemoryStore::new())), }) } /// Construct a MemoryTransport that shares storage with `other`. /// Used by `clone()` and by `MemoryServer` to hand out multiple /// transports sharing a single backing store. pub fn with_shared_store(url: &str, store: Arc>) -> Result { let mut t = Self::new(url)?; t.store = store; Ok(t) } pub fn shared_store(&self) -> Arc> { self.store.clone() } fn abspath_internal(&self, relpath: &UrlFragment) -> Result { let relpath = unescape(relpath).map_err(Error::from)?; if relpath.starts_with('/') { return Ok(relpath); } let cwd_parts = self.cwd.split('/'); let rel_parts = relpath.split('/'); let mut r: Vec = Vec::new(); let store = self.store.lock().unwrap(); for part in cwd_parts.chain(rel_parts) { if part == ".." { if r.is_empty() { return Err(Error::PathNotChild); } r.pop(); } else if part == "." || part.is_empty() { // skip } else { r.push(part.to_string()); // Match Python memory.py _abspath: look up by joined key // without leading slash. Stored symlink keys include a leading // slash, so this effectively never matches; symlink following // happens in resolve_symlinks instead. Kept for byte-for-byte // parity with the Python implementation. let key = r.join("/"); if let Some(target) = store.symlinks.get(&key) { r = target.clone(); } } } Ok(format!("/{}", r.join("/"))) } fn resolve_symlinks(&self, relpath: &UrlFragment) -> Result { let mut path = self.abspath_internal(relpath)?; let store = self.store.lock().unwrap(); while let Some(target) = store.symlinks.get(&path) { path = target.join("/"); if !path.starts_with('/') { path = format!("/{}", path); } } Ok(path) } fn check_parent(store: &MemoryStore, abspath: &str) -> Result<()> { let parent = match abspath.rsplit_once('/') { Some((head, _)) if head.is_empty() => "/".to_string(), Some((head, _)) => head.to_string(), None => "/".to_string(), }; if parent != "/" && !store.dirs.contains_key(&parent) { return Err(Error::NoSuchFile(Some(abspath.to_string()))); } Ok(()) } } struct MemoryReadStream(Cursor>); impl Read for MemoryReadStream { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { self.0.read(buf) } } impl std::io::Seek for MemoryReadStream { fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { self.0.seek(pos) } } impl ReadStream for MemoryReadStream {} /// Write stream that appends straight into the shared MemoryStore so a /// concurrent get_bytes on the same path sees the in-flight bytes /// without an explicit flush — matching the per_transport /// `get_with_open_write_stream_sees_all_content` contract. struct MemoryWriteStream { store: Arc>, abspath: String, } impl Write for MemoryWriteStream { fn write(&mut self, buf: &[u8]) -> std::io::Result { let mut store = self.store.lock().unwrap(); match store.files.get_mut(&self.abspath) { Some((data, _)) => { data.extend_from_slice(buf); Ok(buf.len()) } None => Err(std::io::Error::new( std::io::ErrorKind::NotFound, "memory file removed while write stream was open", )), } } fn flush(&mut self) -> std::io::Result<()> { Ok(()) } } impl WriteStream for MemoryWriteStream { fn sync_data(&self) -> std::io::Result<()> { Ok(()) } } struct MemoryLock { path: String, store: Arc>, } impl Lock for MemoryLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { let mut store = self.store.lock().unwrap(); store.locks.remove(&self.path); Ok(()) } } fn acquire_lock( store: &Arc>, path: &str, ) -> Result> { let mut s = store.lock().unwrap(); if s.locks.contains_key(path) { return Err(Error::LockContention(std::path::PathBuf::from(path))); } s.locks.insert(path.to_string(), ()); Ok(Box::new(MemoryLock { path: path.to_string(), store: store.clone(), })) } impl Transport for MemoryTransport { fn external_url(&self) -> Result { Err(Error::InProcessTransport) } fn can_roundtrip_unix_modebits(&self) -> bool { false } fn base(&self) -> Url { self.base.clone() } fn get(&self, relpath: &UrlFragment) -> Result> { let abspath = self.resolve_symlinks(relpath)?; let store = self.store.lock().unwrap(); if let Some((data, _mode)) = store.files.get(&abspath) { Ok(Box::new(MemoryReadStream(Cursor::new(data.clone())))) } else if store.dirs.contains_key(&abspath) { // Python returns a LateReadError here; we translate that into // an immediate IsADirectoryError for the Rust API. Err(Error::IsADirectoryError(Some(relpath.to_string()))) } else { Err(Error::NoSuchFile(Some(relpath.to_string()))) } } fn has(&self, relpath: &UrlFragment) -> Result { let abspath = self.abspath_internal(relpath)?; let store = self.store.lock().unwrap(); Ok(store.files.contains_key(&abspath) || store.dirs.contains_key(&abspath) || store.symlinks.contains_key(&abspath)) } fn mkdir(&self, relpath: &UrlFragment, permissions: Option) -> Result<()> { let abspath = self.resolve_symlinks(relpath)?; let mut store = self.store.lock().unwrap(); Self::check_parent(&store, &abspath)?; if store.dirs.contains_key(&abspath) { return Err(Error::FileExists(Some(relpath.to_string()))); } store.dirs.insert(abspath, perms_to_mode(permissions)); Ok(()) } fn stat(&self, relpath: &UrlFragment) -> Result { let abspath = self.abspath_internal(relpath)?; let store = self.store.lock().unwrap(); if let Some((data, mode)) = store.files.get(&abspath) { #[cfg(unix)] let stat_mode = (0o100000u32) | mode.unwrap_or(0o644); Ok(Stat { size: data.len(), #[cfg(unix)] mode: stat_mode, kind: FileKind::File, mtime: None, }) } else if let Some(mode) = store.dirs.get(&abspath) { #[cfg(unix)] let stat_mode = (0o040000u32) | mode.unwrap_or(0o755); #[cfg(not(unix))] let _ = mode; Ok(Stat { size: 0, #[cfg(unix)] mode: stat_mode, kind: FileKind::Dir, mtime: None, }) } else if store.symlinks.contains_key(&abspath) { #[cfg(unix)] let stat_mode = 0o120000u32; Ok(Stat { size: 0, #[cfg(unix)] mode: stat_mode, kind: FileKind::Symlink, mtime: None, }) } else { Err(Error::NoSuchFile(Some(abspath))) } } fn clone(&self, offset: Option<&UrlFragment>) -> Result> { let path = crate::urlutils::combine_paths(&self.cwd, offset.unwrap_or("")); let path = if path.is_empty() || !path.ends_with('/') { format!("{}/", path) } else { path }; let url = format!("{}{}", self.scheme, path); let cloned = Self::with_shared_store(&url, self.store.clone())?; Ok(Box::new(cloned)) } fn abspath(&self, relpath: &UrlFragment) -> Result { // Mirror Python: clone(relpath).base, stripping trailing slash unless root. let cloned = self.clone(Some(relpath))?; let s = cloned.base().to_string(); let url_str = if s.matches('/').count() == 3 { s } else { s.trim_end_matches('/').to_string() }; Url::parse(&url_str).map_err(Error::from) } fn relpath(&self, abspath: &Url) -> Result { let base = self.base.as_str(); let target = abspath.as_str(); if let Some(rest) = target.strip_prefix(base) { Ok(rest.to_string()) } else { Err(Error::PathNotChild) } } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, ) -> Result { let abspath = self.resolve_symlinks(relpath)?; let mut buf = Vec::new(); f.read_to_end(&mut buf) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; let mut store = self.store.lock().unwrap(); Self::check_parent(&store, &abspath)?; let len = buf.len() as u64; store .files .insert(abspath, (buf, perms_to_mode(permissions))); Ok(len) } fn delete(&self, relpath: &UrlFragment) -> Result<()> { let abspath = self.abspath_internal(relpath)?; let mut store = self.store.lock().unwrap(); if store.files.remove(&abspath).is_some() { Ok(()) } else if store.symlinks.remove(&abspath).is_some() { Ok(()) } else { Err(Error::NoSuchFile(Some(relpath.to_string()))) } } fn rmdir(&self, relpath: &UrlFragment) -> Result<()> { let abspath = self.resolve_symlinks(relpath)?; let mut store = self.store.lock().unwrap(); if store.files.contains_key(&abspath) { return Err(Error::NotADirectoryError(Some(relpath.to_string()))); } let prefix = format!("{}/", abspath); for path in store.files.keys().chain(store.symlinks.keys()) { if path.starts_with(&prefix) { return Err(Error::DirectoryNotEmptyError(Some(relpath.to_string()))); } } for path in store.dirs.keys() { if path.starts_with(&prefix) && path != &abspath { return Err(Error::DirectoryNotEmptyError(Some(relpath.to_string()))); } } if store.dirs.remove(&abspath).is_none() { return Err(Error::NoSuchFile(Some(relpath.to_string()))); } Ok(()) } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let abs_from = self.resolve_symlinks(rel_from)?; let abs_to = self.resolve_symlinks(rel_to)?; let from_prefix = format!("{}/", abs_from); let replace = |x: &str| -> String { if x == abs_from { abs_to.clone() } else if let Some(rest) = x.strip_prefix(&from_prefix) { format!("{}/{}", abs_to, rest) } else { x.to_string() } }; let mut store = self.store.lock().unwrap(); // Work on copies so rename is atomic on error. let mut files_new = store.files.clone(); let mut symlinks_new = store.symlinks.clone(); let mut dirs_new = store.dirs.clone(); // Collect renames across all three containers, checking for collisions. let mut file_renames: Vec<(String, String)> = Vec::new(); for path in store.files.keys() { let np = replace(path); if np != *path { if files_new.contains_key(&np) { return Err(Error::FileExists(Some(np))); } file_renames.push((path.clone(), np)); } } let mut symlink_renames: Vec<(String, String)> = Vec::new(); for path in store.symlinks.keys() { let np = replace(path); if np != *path { if symlinks_new.contains_key(&np) { return Err(Error::FileExists(Some(np))); } symlink_renames.push((path.clone(), np)); } } let mut dir_renames: Vec<(String, String)> = Vec::new(); for path in store.dirs.keys() { let np = replace(path); if np != *path { if dirs_new.contains_key(&np) { return Err(Error::FileExists(Some(np))); } dir_renames.push((path.clone(), np)); } } for (old, new) in file_renames { let v = files_new.remove(&old).unwrap(); files_new.insert(new, v); } for (old, new) in symlink_renames { let v = symlinks_new.remove(&old).unwrap(); symlinks_new.insert(new, v); } for (old, new) in dir_renames { let v = dirs_new.remove(&old).unwrap(); dirs_new.insert(new, v); } store.files = files_new; store.symlinks = symlinks_new; store.dirs = dirs_new; Ok(()) } fn set_segment_parameter(&mut self, _key: &str, _value: Option<&str>) -> Result<()> { Err(Error::TransportNotPossible) } fn get_segment_parameters(&self) -> Result> { Ok(HashMap::new()) } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, ) -> Result { let abspath = self.resolve_symlinks(relpath)?; let mut buf = Vec::new(); f.read_to_end(&mut buf) .map_err(|e| map_io_err_to_transport_err(e, Some(relpath)))?; let mut store = self.store.lock().unwrap(); Self::check_parent(&store, &abspath)?; let (orig, orig_mode) = store .files .get(&abspath) .cloned() .unwrap_or_else(|| (Vec::new(), None)); let orig_len = orig.len() as u64; let mode = match perms_to_mode(permissions) { Some(m) => Some(m), None => orig_mode, }; let mut combined = orig; combined.extend_from_slice(&buf); store.files.insert(abspath, (combined, mode)); Ok(orig_len) } fn readlink(&self, relpath: &UrlFragment) -> Result { let abspath = self.abspath_internal(relpath)?; let store = self.store.lock().unwrap(); match store.symlinks.get(&abspath) { Some(parts) => Ok(parts.join("/")), None => Err(Error::NoSuchFile(Some(relpath.to_string()))), } } fn hardlink(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(Error::TransportNotPossible) } fn symlink(&self, source: &UrlFragment, link_name: &UrlFragment) -> Result<()> { let abspath = self.abspath_internal(link_name)?; let mut store = self.store.lock().unwrap(); Self::check_parent(&store, &abspath)?; let target: Vec = source.split('/').map(|s| s.to_string()).collect(); store.symlinks.insert(abspath, target); Ok(()) } fn iter_files_recursive(&self) -> Box>> { let store = self.store.lock().unwrap(); let cwd = self.cwd.clone(); let mut results: Vec = Vec::new(); for path in store.files.keys().chain(store.symlinks.keys()) { if path.starts_with(&cwd) { let rest = &path[cwd.len()..]; match escape(rest.as_bytes(), None) { s if !s.is_empty() => results.push(s), _ => {} } } } Box::new(results.into_iter().map(Ok)) } fn open_write_stream( &self, relpath: &UrlFragment, permissions: Option, ) -> Result> { let abspath = self.resolve_symlinks(relpath)?; let mode = perms_to_mode(permissions); // Truncate any existing file and validate the parent exists; this // matches LocalTransport semantics (write streams start empty). { let mut store = self.store.lock().unwrap(); Self::check_parent(&store, &abspath)?; store.files.insert(abspath.clone(), (Vec::new(), mode)); } Ok(Box::new(MemoryWriteStream { store: Arc::clone(&self.store), abspath, })) } fn delete_tree(&self, _relpath: &UrlFragment) -> Result<()> { Err(Error::TransportNotPossible) } fn r#move(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.rename(rel_from, rel_to) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { let abspath = match self.resolve_symlinks(relpath) { Ok(p) => p, Err(e) => return Box::new(std::iter::once(Err(e))), }; let store = self.store.lock().unwrap(); if abspath != "/" && !store.dirs.contains_key(&abspath) { return Box::new(std::iter::once(Err(Error::NoSuchFile(Some( relpath.to_string(), ))))); } let prefix = if abspath.ends_with('/') { abspath.clone() } else { format!("{}/", abspath) }; let mut results: Vec = Vec::new(); for group in [ store.files.keys().collect::>(), store.dirs.keys().collect::>(), store.symlinks.keys().collect::>(), ] { for path in group { if let Some(trailing) = path.strip_prefix(&prefix) { if !trailing.is_empty() && !trailing.contains('/') { results.push(escape(trailing.as_bytes(), None)); } } } } Box::new(results.into_iter().map(Ok)) } fn lock_read(&self, relpath: &UrlFragment) -> Result> { let abspath = self.abspath_internal(relpath)?; acquire_lock(&self.store, &abspath) } fn lock_write(&self, relpath: &UrlFragment) -> Result> { let abspath = self.abspath_internal(relpath)?; acquire_lock(&self.store, &abspath) } fn local_abspath(&self, relpath: &UrlFragment) -> Result { Err(Error::NotLocalUrl(format!("{}{}", self.base, relpath))) } fn get_smart_medium(&self) -> Result> { Err(Error::NoSmartMedium) } fn copy(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { let data = self.get_bytes(rel_from)?; let mut cur = Cursor::new(data); self.put_file(rel_to, &mut cur, None).map(|_| ()) } } #[cfg(test)] mod tests { use super::*; fn t() -> MemoryTransport { MemoryTransport::new("memory:///").unwrap() } #[test] fn new_defaults_and_normalises_url() { let t = MemoryTransport::new("").unwrap(); assert_eq!(t.base().as_str(), "memory:///"); let t = MemoryTransport::new("memory:///foo").unwrap(); assert_eq!(t.base().as_str(), "memory:///foo/"); } #[test] fn put_get_has_and_stat_file() { let t = t(); assert_eq!(t.has("hello").unwrap(), false); t.put_bytes("hello", b"world", None).unwrap(); assert_eq!(t.has("hello").unwrap(), true); assert_eq!(t.get_bytes("hello").unwrap(), b"world"); let st = t.stat("hello").unwrap(); assert_eq!(st.size, 5); assert_eq!(st.kind, FileKind::File); } #[test] fn get_missing_returns_no_such_file() { let t = t(); match t.get_bytes("nope") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn put_file_into_missing_parent_fails() { let t = t(); match t.put_bytes("missing/child", b"x", None) { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn mkdir_and_list_dir() { let t = t(); t.mkdir("d", None).unwrap(); t.put_bytes("d/a", b"1", None).unwrap(); t.put_bytes("d/b", b"22", None).unwrap(); let mut entries: Vec = t.list_dir("d").filter_map(|r| r.ok()).collect(); entries.sort(); assert_eq!(entries, vec!["a".to_string(), "b".to_string()]); assert_eq!(t.stat("d").unwrap().kind, FileKind::Dir); } #[test] fn mkdir_existing_fails() { let t = t(); t.mkdir("d", None).unwrap(); match t.mkdir("d", None) { Err(Error::FileExists(_)) => {} other => panic!("expected FileExists, got {:?}", other), } } #[test] fn delete_and_rmdir() { let t = t(); t.mkdir("d", None).unwrap(); t.put_bytes("d/f", b"x", None).unwrap(); t.delete("d/f").unwrap(); assert_eq!(t.has("d/f").unwrap(), false); t.rmdir("d").unwrap(); assert_eq!(t.has("d").unwrap(), false); } #[test] fn rmdir_nonempty_fails() { let t = t(); t.mkdir("d", None).unwrap(); t.put_bytes("d/f", b"x", None).unwrap(); match t.rmdir("d") { Err(Error::DirectoryNotEmptyError(_)) => {} other => panic!("expected DirectoryNotEmptyError, got {:?}", other), } } #[test] fn rename_file() { let t = t(); t.put_bytes("a", b"hi", None).unwrap(); t.rename("a", "b").unwrap(); assert_eq!(t.has("a").unwrap(), false); assert_eq!(t.get_bytes("b").unwrap(), b"hi"); } #[test] fn append_file_extends_content_and_returns_offset() { let t = t(); t.put_bytes("f", b"abc", None).unwrap(); let mut more = Cursor::new(b"DEF".to_vec()); let offset = t.append_file("f", &mut more, None).unwrap(); assert_eq!(offset, 3); assert_eq!(t.get_bytes("f").unwrap(), b"abcDEF"); } #[test] fn symlink_and_readlink() { let t = t(); t.put_bytes("target", b"data", None).unwrap(); t.symlink("target", "link").unwrap(); assert_eq!(t.readlink("link").unwrap(), "target"); assert_eq!(t.stat("link").unwrap().kind, FileKind::Symlink); assert_eq!(t.get_bytes("link").unwrap(), b"data"); } #[test] fn lock_read_contention() { let t = t(); t.put_bytes("f", b"", None).unwrap(); let _l = t.lock_read("f").ok().expect("first lock"); match t.lock_read("f") { Err(Error::LockContention(_)) => {} Err(other) => panic!("expected LockContention, got {:?}", other), Ok(_) => panic!("expected LockContention, got Ok"), } } #[test] fn lock_release_allows_reacquire() { let t = t(); t.put_bytes("f", b"", None).unwrap(); { let mut l = t.lock_read("f").ok().expect("first lock"); l.unlock().ok().expect("unlock"); } let _l2 = t.lock_read("f").ok().expect("reacquire"); } #[test] fn clone_shares_storage() { let t = t(); t.mkdir("sub", None).unwrap(); let c = t.clone(Some("sub")).unwrap(); t.put_bytes("sub/f", b"shared", None).unwrap(); assert_eq!(c.get_bytes("f").unwrap(), b"shared"); } #[test] fn external_url_errors_in_process() { let t = t(); match t.external_url() { Err(Error::InProcessTransport) => {} other => panic!("expected InProcessTransport, got {:?}", other), } } #[test] fn iter_files_recursive_lists_files_and_symlinks() { let t = t(); t.mkdir("d", None).unwrap(); t.put_bytes("d/a", b"1", None).unwrap(); t.put_bytes("d/b", b"2", None).unwrap(); t.symlink("d/a", "d/link").unwrap(); let sub = t.clone(Some("d")).unwrap(); let mut files: Vec = sub.iter_files_recursive().filter_map(|r| r.ok()).collect(); files.sort(); assert_eq!( files, vec!["a".to_string(), "b".to_string(), "link".to_string()] ); } #[test] fn open_write_stream_round_trip() { let t = t(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"hello ").unwrap(); stream.write_all(b"world").unwrap(); drop(stream); assert_eq!(t.get_bytes("w").unwrap(), b"hello world"); } #[test] fn open_write_stream_visible_without_flush() { // The per_transport contract: a concurrent get_bytes after write // (no explicit flush) sees the in-flight bytes. let t = t(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"abc").unwrap(); assert_eq!(t.get_bytes("w").unwrap(), b"abc"); stream.write_all(b"def").unwrap(); assert_eq!(t.get_bytes("w").unwrap(), b"abcdef"); } #[test] fn open_write_stream_truncates_existing() { let t = t(); t.put_bytes("w", b"old contents", None).unwrap(); let mut stream = t.open_write_stream("w", None).unwrap(); stream.write_all(b"new").unwrap(); drop(stream); assert_eq!(t.get_bytes("w").unwrap(), b"new"); } #[test] fn open_write_stream_rejects_missing_parent() { let t = t(); match t.open_write_stream("missing/child", None) { Ok(_) => panic!("expected NoSuchFile, got Ok"), Err(Error::NoSuchFile(_)) => {} Err(other) => panic!("expected NoSuchFile, got {:?}", other), } } } dromedary-0.1.1/src/osutils/mod.rs000064400000000000000000000077431046102023000152420ustar 00000000000000pub mod path; /// Transport-style path helpers ported from dromedary/osutils.py. /// /// These operate on forward-slash URL-style paths (strings), not on /// native OS paths. See `path` for OS-path helpers. /// Split a forward-slash path into its components. /// /// Leading and trailing slashes are stripped. `""` and `"/"` both return /// an empty vector. pub fn splitpath(path: &str) -> Vec { if path.is_empty() || path == "/" { return Vec::new(); } let trimmed = path.trim_start_matches('/').trim_end_matches('/'); if trimmed.is_empty() { return Vec::new(); } trimmed.split('/').map(|s| s.to_string()).collect() } /// Join forward-slash path components. /// /// Empty and `"."` components are dropped. If the first component starts /// with `/`, the result is absolute. pub fn pathjoin(parts: &[&str]) -> String { if parts.is_empty() { return String::new(); } let absolute = parts[0].starts_with('/'); let components: Vec<&str> = parts .iter() .copied() .filter(|p| !p.is_empty() && *p != ".") .collect(); if components.is_empty() { return String::new(); } let joined = components .iter() .map(|c| c.trim_start_matches('/')) .collect::>() .join("/"); if absolute { format!("/{}", joined) } else { joined } } /// File kinds reported by `file_kind_from_stat_mode`. String values match /// the Python implementation for byte-for-byte parity. pub const KIND_FILE: &str = "file"; pub const KIND_DIRECTORY: &str = "directory"; pub const KIND_SYMLINK: &str = "symlink"; pub const KIND_CHARDEV: &str = "chardev"; pub const KIND_BLOCK: &str = "block"; pub const KIND_FIFO: &str = "fifo"; pub const KIND_SOCKET: &str = "socket"; pub const KIND_UNKNOWN: &str = "unknown"; /// Translate a Unix stat mode into a kind string matching Python's /// `stat.S_IS*` classification. pub fn file_kind_from_stat_mode(stat_mode: u32) -> &'static str { const S_IFMT: u32 = 0o170000; const S_IFREG: u32 = 0o100000; const S_IFDIR: u32 = 0o040000; const S_IFLNK: u32 = 0o120000; const S_IFCHR: u32 = 0o020000; const S_IFBLK: u32 = 0o060000; const S_IFIFO: u32 = 0o010000; const S_IFSOCK: u32 = 0o140000; match stat_mode & S_IFMT { S_IFREG => KIND_FILE, S_IFDIR => KIND_DIRECTORY, S_IFLNK => KIND_SYMLINK, S_IFCHR => KIND_CHARDEV, S_IFBLK => KIND_BLOCK, S_IFIFO => KIND_FIFO, S_IFSOCK => KIND_SOCKET, _ => KIND_UNKNOWN, } } #[cfg(test)] mod tests { use super::*; #[test] fn splitpath_basics() { assert_eq!(splitpath(""), Vec::::new()); assert_eq!(splitpath("/"), Vec::::new()); assert_eq!(splitpath("a"), vec!["a".to_string()]); assert_eq!(splitpath("/a"), vec!["a".to_string()]); assert_eq!(splitpath("a/"), vec!["a".to_string()]); assert_eq!( splitpath("/a/b/c"), vec!["a".to_string(), "b".to_string(), "c".to_string()] ); } #[test] fn pathjoin_basics() { assert_eq!(pathjoin(&[]), ""); assert_eq!(pathjoin(&["a", "b"]), "a/b"); assert_eq!(pathjoin(&["a", "", "b"]), "a/b"); assert_eq!(pathjoin(&["a", ".", "b"]), "a/b"); assert_eq!(pathjoin(&["/a", "b"]), "/a/b"); assert_eq!(pathjoin(&["", ""]), ""); } #[test] fn file_kind_from_stat_mode_regular() { assert_eq!(file_kind_from_stat_mode(0o100644), KIND_FILE); assert_eq!(file_kind_from_stat_mode(0o040755), KIND_DIRECTORY); assert_eq!(file_kind_from_stat_mode(0o120777), KIND_SYMLINK); } #[test] fn file_kind_from_stat_mode_special() { assert_eq!(file_kind_from_stat_mode(0o020000), KIND_CHARDEV); assert_eq!(file_kind_from_stat_mode(0o060000), KIND_BLOCK); assert_eq!(file_kind_from_stat_mode(0o010000), KIND_FIFO); assert_eq!(file_kind_from_stat_mode(0o140000), KIND_SOCKET); } } dromedary-0.1.1/src/osutils/path.rs000064400000000000000000000153631046102023000154140ustar 00000000000000use std::path::{Component, Path, PathBuf}; pub mod win32 { use lazy_static::lazy_static; use regex::Regex; use std::path::{Path, PathBuf}; pub fn fixdrive(path: &Path) -> PathBuf { if path.as_os_str().len() < 2 || path.to_str().unwrap().chars().nth(1).unwrap() != ':' { return path.into(); } if let Some(drive) = path.as_os_str().to_str().unwrap().get(..2) { PathBuf::from(drive.to_uppercase() + path.to_str().unwrap().get(2..).unwrap()) } else { path.into() } } pub fn fix_separators(path: &Path) -> PathBuf { if path.to_path_buf().to_str().unwrap().contains('\\') { path.to_path_buf() .to_str() .unwrap() .replace('\\', "/") .into() } else { path.into() } } lazy_static! { static ref ABS_WINDOWS_PATH_RE: Regex = Regex::new(r#"^[A-Za-z]:[/\\]"#).unwrap(); } pub fn abspath(path: &Path) -> Result { #[cfg(not(windows))] if ABS_WINDOWS_PATH_RE.is_match(path.to_str().unwrap()) { return Ok(path.to_path_buf()); } use path_clean::PathClean; let cwd = std::env::current_dir()?; let ap = cwd.join(path).clean(); Ok(fixdrive(&fix_separators(ap.as_path()))) } /// Resolve a Windows path, following symlinks and junctions. /// /// The `std::fs::canonicalize` Windows implementation already does the /// right thing (it issues `GetFinalPathNameByHandleW`), so we just delegate /// and normalize the forward-slash convention the rest of this module /// relies on. pub fn realpath(path: &Path) -> Result { let canonical = std::fs::canonicalize(path)?; Ok(fixdrive(&fix_separators(canonical.as_path()))) } } pub mod posix { use std::collections::HashMap; use std::path::{Component, Path, PathBuf}; pub fn abspath(path: &Path) -> Result { use path_clean::PathClean; if path.is_absolute() { return Ok(path.to_path_buf()); } let cwd = std::env::current_dir()?; let ap = cwd.join(path).clean(); Ok(ap.as_path().to_path_buf()) } pub fn realpath>(filename: P) -> std::io::Result { let filename = filename.as_ref().to_path_buf(); let (path, _) = join_realpath(Path::new(""), &filename, &mut HashMap::new())?; abspath(path.as_path()) } fn join_realpath( path: &Path, rest: &Path, seen: &mut HashMap>, ) -> std::io::Result<(PathBuf, bool)> { let rest = rest.to_path_buf(); let mut path = path.to_path_buf(); let mut components = rest.components(); while let Some(component) = components.next() { match component { Component::RootDir => { path = PathBuf::from("/"); } Component::CurDir | Component::Prefix(_) => {} Component::ParentDir => { if path.components().next().is_none() { path = PathBuf::from(".."); } else if path.file_name().unwrap() == ".." { path = path.join(".."); } else { path = path.parent().unwrap().to_path_buf(); } } Component::Normal(name) => { let mut newpath = path.join(name); let st = std::fs::symlink_metadata(&newpath); let is_link = st.is_ok() && st.unwrap().file_type().is_symlink(); if !is_link { path = newpath; } else if let Some(cached) = seen.get(&newpath) { match cached { Some(target) => { path = target.clone(); } None => { return Ok((newpath, false)); } } } else { seen.insert(newpath.clone(), None); let ok; (path, ok) = join_realpath( path.as_path(), std::fs::read_link(&newpath)?.as_path(), seen, )?; if !ok { components.for_each(|c| newpath.push(c)); return Ok((newpath, false)); } seen.insert(newpath, Some(path.clone())); } } } } Ok((path.to_path_buf(), true)) } } pub fn abspath(path: &Path) -> Result { #[cfg(windows)] return win32::abspath(path); #[cfg(not(windows))] return posix::abspath(path); } pub fn normpath>(path: P) -> PathBuf { let mut stack = Vec::new(); for component in path.as_ref().components() { match component { Component::Prefix(_) => { stack.clear(); stack.push(component.as_os_str()); } Component::RootDir => { stack.clear(); stack.push(component.as_os_str()); } Component::CurDir => {} Component::ParentDir => { if stack.len() > 1 { stack.pop(); } } Component::Normal(c) => { stack.push(c); } } } let mut result = PathBuf::new(); for c in stack { result.push(c); } result } #[cfg(not(windows))] pub const MIN_ABS_PATHLENGTH: usize = 1; #[cfg(windows)] pub const MIN_ABS_PATHLENGTH: usize = 3; pub fn relpath(base: &Path, path: &Path) -> Option { if base.to_str().unwrap().len() < MIN_ABS_PATHLENGTH { return None; } let rp = abspath(path).unwrap(); let mut s = Vec::new(); let mut head = rp.as_path(); let mut tail; loop { if head.as_os_str().len() <= base.as_os_str().len() && head != base { return None; } if head == base { break; } (head, tail) = (head.parent().unwrap(), head.file_name().unwrap()); if !tail.is_empty() { s.push(tail); } } Some(s.into_iter().rev().collect::()) } pub fn realpath(f: &Path) -> std::io::Result { #[cfg(windows)] return win32::realpath(f); #[cfg(not(windows))] return posix::realpath(f); } dromedary-0.1.1/src/pathfilter.rs000064400000000000000000000276651046102023000151300ustar 00000000000000//! Path-filtering Transport decorator, ported from dromedary/pathfilter.py. //! //! Wraps a backing transport and passes every relpath through a filter //! function before delegating. The filter rebases relpaths against a //! "server root" path derived from the transport's base URL. An optional //! user-supplied callable can further rewrite paths (chroot omits it). use crate::lock::Lock; use crate::urlutils::combine_paths; use crate::{Error, ReadStream, Result, SmartMedium, Stat, Transport, UrlFragment, WriteStream}; use std::collections::HashMap; use std::fs::Permissions; use url::Url; pub type FilterFunc = Box String + Send + Sync>; pub struct PathFilteringTransport { backing: Box, base_path: String, scheme: String, base: Url, filter_func: Option, } impl PathFilteringTransport { /// Construct a PathFilteringTransport. /// /// `scheme` is the URL scheme this transport exposes (e.g. "filtered-42:///" /// or "chroot-42:///"), `base_path` is the path portion of the transport's /// base URL (must start with `/`), and `filter_func` is an optional /// rewriter applied after the server-root rebase. pub fn new( backing: Box, scheme: impl Into, base_path: impl Into, filter_func: Option, ) -> Result { let scheme = scheme.into(); let mut base_path = base_path.into(); if !base_path.starts_with('/') { return Err(Error::PathNotChild); } if !base_path.ends_with('/') { base_path.push('/'); } // scheme is expected to end with ":///"; base_path starts with "/", // so join by stripping the leading slash of base_path. let base_url = format!("{}{}", scheme, &base_path[1..]); let base = Url::parse(&base_url).map_err(Error::from)?; Ok(Self { backing, base_path, scheme, base, filter_func, }) } pub fn relpath_from_server_root(&self, relpath: &str) -> Result { let unfiltered = combine_paths(&self.base_path, relpath); if !unfiltered.starts_with('/') { return Err(Error::PathNotChild); } Ok(unfiltered[1..].to_string()) } pub fn filter(&self, relpath: &str) -> Result { let p = self.relpath_from_server_root(relpath)?; match &self.filter_func { Some(f) => Ok(f(&p)), None => Ok(p), } } } impl std::fmt::Debug for PathFilteringTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "PathFilteringTransport({})", self.base) } } impl Transport for PathFilteringTransport { fn external_url(&self) -> Result { self.backing.external_url() } fn can_roundtrip_unix_modebits(&self) -> bool { self.backing.can_roundtrip_unix_modebits() } fn base(&self) -> Url { self.base.clone() } fn is_readonly(&self) -> bool { self.backing.is_readonly() } fn listable(&self) -> bool { self.backing.listable() } fn get(&self, relpath: &UrlFragment) -> Result> { self.backing.get(&self.filter(relpath)?) } fn has(&self, relpath: &UrlFragment) -> Result { self.backing.has(&self.filter(relpath)?) } fn stat(&self, relpath: &UrlFragment) -> Result { self.backing.stat(&self.filter(relpath)?) } fn clone(&self, offset: Option<&UrlFragment>) -> Result> { // Clone of a path-filtering transport returns a backing-transport clone // rebased to the filtered offset. This matches Python's // `self.__class__(self.server, self.abspath(relpath))` well enough for // in-process use, but loses the filter wrapping. Callers that need a // filtered clone should reconstruct one. let target_relpath = match offset { Some(o) => self.relpath_from_server_root(o)?, None => self.relpath_from_server_root("")?, }; self.backing.clone(Some(&target_relpath)) } fn abspath(&self, relpath: &UrlFragment) -> Result { // Deliberately unfiltered: filtering happens when the base is // resolved against the backing transport, not at abspath time. let p = self.relpath_from_server_root(relpath)?; let url = format!("{}{}", self.scheme, p); Url::parse(&url).map_err(Error::from) } fn relpath(&self, abspath: &Url) -> Result { let base = self.base.as_str(); let target = abspath.as_str(); target .strip_prefix(base) .map(|s| s.to_string()) .ok_or(Error::PathNotChild) } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn std::io::Read, permissions: Option, ) -> Result { self.backing .put_file(&self.filter(relpath)?, f, permissions) } fn mkdir(&self, relpath: &UrlFragment, permissions: Option) -> Result<()> { self.backing.mkdir(&self.filter(relpath)?, permissions) } fn delete(&self, relpath: &UrlFragment) -> Result<()> { self.backing.delete(&self.filter(relpath)?) } fn rmdir(&self, relpath: &UrlFragment) -> Result<()> { self.backing.rmdir(&self.filter(relpath)?) } fn rename(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.backing .rename(&self.filter(rel_from)?, &self.filter(rel_to)?) } fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> Result<()> { self.backing.set_segment_parameter(key, value) } fn get_segment_parameters(&self) -> Result> { self.backing.get_segment_parameters() } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn std::io::Read, permissions: Option, ) -> Result { self.backing .append_file(&self.filter(relpath)?, f, permissions) } fn readlink(&self, relpath: &UrlFragment) -> Result { self.backing.readlink(&self.filter(relpath)?) } fn hardlink(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.backing .hardlink(&self.filter(rel_from)?, &self.filter(rel_to)?) } fn symlink(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.backing .symlink(&self.filter(rel_from)?, &self.filter(rel_to)?) } fn iter_files_recursive(&self) -> Box>> { // Clone the backing transport to the filtered "." path and let it // walk from there. let filtered = match self.filter(".") { Ok(p) => p, Err(e) => return Box::new(std::iter::once(Err(e))), }; match self.backing.clone(Some(&filtered)) { Ok(cloned) => cloned.iter_files_recursive(), Err(e) => Box::new(std::iter::once(Err(e))), } } fn open_write_stream( &self, relpath: &UrlFragment, permissions: Option, ) -> Result> { self.backing .open_write_stream(&self.filter(relpath)?, permissions) } fn delete_tree(&self, relpath: &UrlFragment) -> Result<()> { self.backing.delete_tree(&self.filter(relpath)?) } fn r#move(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.backing .r#move(&self.filter(rel_from)?, &self.filter(rel_to)?) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { match self.filter(relpath) { Ok(p) => self.backing.list_dir(&p), Err(e) => Box::new(std::iter::once(Err(e))), } } fn lock_read(&self, relpath: &UrlFragment) -> Result> { self.backing.lock_read(&self.filter(relpath)?) } fn lock_write(&self, relpath: &UrlFragment) -> Result> { self.backing.lock_write(&self.filter(relpath)?) } fn local_abspath(&self, relpath: &UrlFragment) -> Result { self.backing.local_abspath(&self.filter(relpath)?) } fn get_smart_medium(&self) -> Result> { self.backing.get_smart_medium() } fn copy(&self, rel_from: &UrlFragment, rel_to: &UrlFragment) -> Result<()> { self.backing .copy(&self.filter(rel_from)?, &self.filter(rel_to)?) } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn backing_with(files: &[(&str, &[u8])]) -> Box { let mem = MemoryTransport::new("memory:///").unwrap(); for (p, data) in files { // Ensure parents exist. if let Some(parent) = std::path::Path::new(p).parent() { let parent = parent.to_string_lossy().to_string(); if !parent.is_empty() { let _ = mem.mkdir(&parent, None); } } mem.put_bytes(p, data, None).unwrap(); } Box::new(mem) } fn make(base_path: &str, filter: Option) -> PathFilteringTransport { let backing = backing_with(&[("a", b"A"), ("sub/b", b"B")]); PathFilteringTransport::new(backing, "filtered-1:///", base_path, filter).unwrap() } #[test] fn pass_through_filter_none() { let t = make("/", None); assert_eq!(t.get_bytes("a").unwrap(), b"A"); assert_eq!(t.has("sub/b").unwrap(), true); } #[test] fn rebases_under_subdirectory() { let t = make("/sub/", None); assert_eq!(t.get_bytes("b").unwrap(), b"B"); match t.get_bytes("a") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } #[test] fn filter_func_rewrites_path() { // Filter prepends "sub/" to every relpath; starting from root that // means every get effectively reads from /sub/... let filter: FilterFunc = Box::new(|p: &str| format!("sub/{}", p)); let t = make("/", Some(filter)); assert_eq!(t.get_bytes("b").unwrap(), b"B"); } #[test] fn mkdir_and_delete_round_trip() { let t = make("/", None); t.mkdir("new", None).unwrap(); t.put_bytes("new/f", b"x", None).unwrap(); assert_eq!(t.get_bytes("new/f").unwrap(), b"x"); t.delete("new/f").unwrap(); assert_eq!(t.has("new/f").unwrap(), false); t.rmdir("new").unwrap(); } #[test] fn list_dir_passes_through() { let t = make("/", None); let mut entries: Vec = t.list_dir("sub").filter_map(|r| r.ok()).collect(); entries.sort(); assert_eq!(entries, vec!["b".to_string()]); } #[test] fn iter_files_recursive_uses_filtered_root() { let t = make("/sub/", None); let mut files: Vec = t.iter_files_recursive().filter_map(|r| r.ok()).collect(); files.sort(); assert_eq!(files, vec!["b".to_string()]); } #[test] fn abspath_is_not_filtered() { let filter: FilterFunc = Box::new(|p: &str| format!("sub/{}", p)); let t = make("/", Some(filter)); let u = t.abspath("x").unwrap(); assert!(u.as_str().ends_with("/x"), "got {}", u); } #[test] fn is_readonly_forwards() { let t = make("/", None); assert_eq!(t.is_readonly(), false); } #[test] fn base_path_must_start_with_slash() { let backing = backing_with(&[]); match PathFilteringTransport::new(backing, "filtered-1:///", "sub", None) { Err(Error::PathNotChild) => {} other => panic!("expected PathNotChild, got {:?}", other), } } } dromedary-0.1.1/src/pyo3.rs000064400000000000000000000626241046102023000136520ustar 00000000000000use crate::{ Error, Lock, LockError, ReadStream, Result, SmartMedium, Stat, Transport, Url, UrlFragment, WriteStream, }; use pyo3::import_exception; use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3_filelike::PyBinaryFile; use std::collections::HashMap; use std::fs::Permissions; use std::io::{Read, Write}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; /// Convert `Option` to the `Optional[int]` mode expected by the /// Python transport API. On Windows Python itself ignores the mode argument /// (chmod there only touches the read-only bit), so we pass `None`. #[inline] fn perms_to_py_mode(perms: Option<&Permissions>) -> Option { #[cfg(unix)] { perms.map(|p| p.mode()) } #[cfg(not(unix))] { let _ = perms; None } } import_exception!(dromedary.errors, TransportError); import_exception!(dromedary.errors, NoSmartMedium); import_exception!(dromedary.errors, InProcessTransport); import_exception!(dromedary.errors, NotLocalUrl); import_exception!(dromedary.errors, NoSuchFile); import_exception!(dromedary.errors, FileExists); import_exception!(dromedary.errors, TransportNotPossible); import_exception!(dromedary.errors, UrlError); import_exception!(dromedary.errors, PermissionDenied); import_exception!(dromedary.errors, PathNotChild); import_exception!(dromedary.errors, ShortReadvError); import_exception!(dromedary.errors, LockContention); import_exception!(dromedary.errors, LockFailed); import_exception!(dromedary.errors, DirectoryNotEmpty); import_exception!(dromedary.errors, NotADirectory); import_exception!(dromedary.errors, ResourceBusy); import_exception!(dromedary.errors, ReadError); import_exception!(dromedary.urlutils, InvalidURL); struct PySmartMedium(Py); impl SmartMedium for PySmartMedium {} pub struct PyTransport(Py); impl From> for PyTransport { fn from(obj: Py) -> Self { PyTransport(obj) } } fn map_py_err_to_lock_err(e: PyErr) -> LockError { Python::attach(|py| { if e.is_instance_of::(py) { LockError::Contention(e.value(py).getattr("lock").unwrap().extract().unwrap()) } else if e.is_instance_of::(py) { let v = e.value(py); LockError::Failed( v.getattr("lock").unwrap().extract().unwrap(), v.getattr("why").unwrap().extract().unwrap(), ) } else { LockError::IoError(e.into()) } }) } struct PyLock(Py); impl Lock for PyLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { Python::attach(|py| { self.0 .call_method0(py, "unlock") .map_err(map_py_err_to_lock_err)?; Ok(()) }) } } impl<'py> IntoPyObject<'py> for PyTransport { type Target = PyAny; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> std::result::Result { Ok(self.0.bind(py).clone()) } } impl From for Error { fn from(e: PyErr) -> Self { Python::attach(|py| { let arg = |_i| -> Option { let args = e.value(py).getattr("args").ok()?; let item = args.get_item(0).ok()?; if item.is_none() { None } else { item.extract::().ok() } }; if e.is_instance_of::(py) { Error::InProcessTransport } else if e.is_instance_of::(py) { let url = e .value(py) .getattr("url") .ok() .and_then(|u| u.extract::().ok()) .unwrap_or_default(); Error::NotLocalUrl(url) } else if e.is_instance_of::(py) { Error::NoSuchFile(arg(0)) } else if e.is_instance_of::(py) { Error::FileExists(arg(0)) } else if e.is_instance_of::(py) { Error::TransportNotPossible } else if e.is_instance_of::(py) { Error::PermissionDenied(arg(0)) } else if e.is_instance_of::(py) { Error::PathNotChild } else if e.is_instance_of::(py) { Error::DirectoryNotEmptyError(arg(0)) } else if e.is_instance_of::(py) { Error::NotADirectoryError(arg(0)) } else if e.is_instance_of::(py) { Error::ResourceBusy(arg(0)) } else if e.is_instance_of::(py) { Error::IsADirectoryError(arg(0)) } else if e.is_instance_of::(py) { Error::UrlutilsError(crate::urlutils::Error::UrlNotAscii( arg(0).unwrap_or_default(), )) } else if e.is_instance_of::(py) { let value = e.value(py); Error::ShortReadvError( value.getattr("path").unwrap().extract::().unwrap(), value.getattr("offset").unwrap().extract::().unwrap(), value.getattr("length").unwrap().extract::().unwrap(), value.getattr("actual").unwrap().extract::().unwrap(), ) } else { // Don't panic on unrecognised exception types — funnel them // through Error::Io so the caller sees something useful and // the worker stays alive. New variants should still be added // explicitly above when they have a real semantic mapping. Error::Io(std::io::Error::other(e.to_string())) } }) } } impl ReadStream for PyBinaryFile {} // Bit of a hack - this reads the entire buffer, and then streams it fn py_read(r: &mut dyn Read) -> PyResult> { Python::attach(|py| { let mut buffer = Vec::new(); r.read_to_end(&mut buffer)?; let io = py.import("io")?; let bytesio = io.getattr("BytesIO")?; Ok(bytesio.call1((buffer,))?.unbind()) }) } struct PyWriteStream(Py); impl Write for PyWriteStream { fn write(&mut self, buf: &[u8]) -> std::io::Result { Python::attach(|py| { let obj = self.0.call_method1(py, "write", (buf,))?; Ok(obj.extract::(py)?) }) } fn flush(&mut self) -> std::io::Result<()> { Python::attach(|py| { self.0.call_method0(py, "flush")?; Ok(()) }) } } impl WriteStream for PyWriteStream { fn sync_data(&self) -> std::io::Result<()> { Python::attach(|py| { self.0.call_method0(py, "fdatasync")?; Ok(()) }) } } impl std::fmt::Debug for PyTransport { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "PyTransport({:?})", self.0) } } impl Transport for PyTransport { fn external_url(&self) -> Result { Python::attach(|py| { let obj = self.0.call_method0(py, "external_url")?; let s = obj.extract::(py)?; Url::parse(&s).map_err(Error::from) }) } fn get_bytes(&self, path: &str) -> Result> { Python::attach(|py| { let obj = self.0.call_method1(py, "get_bytes", (path,))?; let bytes = obj.cast_bound::(py).map_err(PyErr::from)?; Ok(bytes.as_bytes().to_vec()) }) } fn get(&self, path: &str) -> Result> { Python::attach(|py| { let obj = self.0.call_method1(py, "get", (path,))?; Ok(Box::new(PyBinaryFile::from(obj)) as Box) }) } fn base(&self) -> Url { Python::attach(|py| { // `.base` is a required attribute on every Transport. If the // wrapped Python object can't produce one we have no choice but // to fall back to a placeholder URL — none of the trait callers // expect this to fail. let url_str = self .0 .getattr(py, "base") .ok() .and_then(|obj| obj.extract::(py).ok()) .unwrap_or_default(); Url::parse(&url_str).unwrap_or_else(|_| Url::parse("file:///").unwrap()) }) } fn has(&self, path: &UrlFragment) -> Result { Python::attach(|py| { let obj = self.0.call_method1(py, "has", (path,))?; Ok(obj.extract::(py)?) }) } fn has_any(&self, paths: &[&UrlFragment]) -> Result { Python::attach(|py| { let obj = self.0.call_method1(py, "has_any", (paths.to_vec(),))?; Ok(obj.extract::(py)?) }) } fn mkdir(&self, relpath: &UrlFragment, perms: Option) -> Result<()> { Python::attach(|py| { self.0 .call_method1(py, "mkdir", (relpath, perms_to_py_mode(perms.as_ref())))?; Ok(()) }) } fn ensure_base(&self, perms: Option) -> Result { Python::attach(|py| { let obj = self.0 .call_method1(py, "ensure_base", (perms_to_py_mode(perms.as_ref()),))?; Ok(obj.extract::(py)?) }) } fn stat(&self, path: &UrlFragment) -> Result { Python::attach(|py| { let stat_result = self.0.call_method1(py, "stat", (path,))?; let mtime = if let Ok(mtime) = stat_result.getattr(py, "mtime") { Some(mtime.extract::(py)?) } else { None }; let st_mode = stat_result.getattr(py, "st_mode")?.extract::(py)?; // Derive kind from the POSIX mode bits Python reported. // On Windows Python still reports something meaningful for dir vs file. let kind = { const S_IFMT: u32 = 0o170000; const S_IFDIR: u32 = 0o040000; const S_IFREG: u32 = 0o100000; const S_IFLNK: u32 = 0o120000; match st_mode & S_IFMT { S_IFDIR => crate::FileKind::Dir, S_IFREG => crate::FileKind::File, S_IFLNK => crate::FileKind::Symlink, _ => crate::FileKind::Other, } }; Ok(Stat { #[cfg(unix)] mode: st_mode, kind, size: stat_result.getattr(py, "st_size")?.extract::(py)?, mtime, }) }) } fn clone(&self, path: Option<&UrlFragment>) -> Result> { Python::attach(|py| { let obj = self.0.call_method1(py, "clone", (path,))?; let transport: Box = Box::new(PyTransport(obj)); Ok(transport) }) } fn relpath(&self, path: &Url) -> Result { Python::attach(|py| { let obj = self.0.call_method1(py, "relpath", (path.to_string(),))?; Ok(obj.extract::(py)?) }) } fn abspath(&self, relpath: &UrlFragment) -> Result { let s = Python::attach(|py| { let obj = self.0.call_method1(py, "abspath", (relpath,))?; obj.extract::(py) })?; Url::parse(&s).map_err(Error::from) } fn put_file( &self, relpath: &UrlFragment, f: &mut dyn Read, mode: Option, ) -> Result { let f = py_read(f)?; Python::attach(|py| { let ret = self.0.call_method1( py, "put_file", (relpath, f, perms_to_py_mode(mode.as_ref())), )?; Ok(ret.extract::(py)?) }) } fn put_bytes( &self, relpath: &UrlFragment, bytes: &[u8], mode: Option, ) -> Result<()> { Python::attach(|py| { self.0.call_method1( py, "put_bytes", (relpath, bytes, perms_to_py_mode(mode.as_ref())), )?; Ok(()) }) } fn put_file_non_atomic( &self, relpath: &UrlFragment, f: &mut dyn Read, mode: Option, create_parent: Option, parent_mode: Option, ) -> Result<()> { let f = py_read(f)?; Python::attach(|py| { self.0.call_method1( py, "put_file_non_atomic", ( relpath, f, perms_to_py_mode(mode.as_ref()), create_parent, perms_to_py_mode(parent_mode.as_ref()), ), )?; Ok(()) }) } fn put_bytes_non_atomic( &self, relpath: &UrlFragment, bytes: &[u8], mode: Option, create_parent: Option, parent_mode: Option, ) -> Result<()> { Python::attach(|py| { self.0.call_method1( py, "put_bytes_non_atomic", ( relpath, bytes, perms_to_py_mode(mode.as_ref()), create_parent, perms_to_py_mode(parent_mode.as_ref()), ), )?; Ok(()) }) } fn delete(&self, relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "delete", (relpath,))?; Ok(()) }) } fn rmdir(&self, relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "rmdir", (relpath,))?; Ok(()) }) } fn rename(&self, relpath: &UrlFragment, new_relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "rename", (relpath, new_relpath))?; Ok(()) }) } fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> Result<()> { Python::attach(|py| { self.0 .call_method1(py, "set_segment_parameter", (key, value))?; Ok(()) }) } fn get_segment_parameters(&self) -> Result> { Python::attach(|py| { Ok(self .0 .call_method0(py, "get_segment_parameters")? .extract::>(py)?) }) } fn create_prefix(&self, permissions: Option) -> Result<()> { Python::attach(|py| { self.0.call_method1( py, "create_prefix", (perms_to_py_mode(permissions.as_ref()),), )?; Ok(()) }) } fn recommended_page_size(&self) -> usize { Python::attach(|py| { self.0 .call_method0(py, "recommended_page_size") .ok() .and_then(|obj| obj.extract::(py).ok()) .unwrap_or(4 * 1024) }) } fn is_readonly(&self) -> bool { Python::attach(|py| { self.0 .call_method0(py, "is_readonly") .ok() .and_then(|obj| obj.extract::(py).ok()) .unwrap_or(false) }) } fn readv( &self, relpath: &UrlFragment, offsets: Vec<(u64, usize)>, adjust_for_latency: bool, upper_limit: Option, ) -> Box)>> + Send> { let iter = Python::attach(|py| -> Result> { let raw = self.0.call_method1( py, "readv", (relpath, offsets, adjust_for_latency, upper_limit), )?; let it = raw.bind(py).try_iter().map_err(Error::from)?; Ok(it.unbind().into_any()) }); let iter = match iter { Ok(i) => i, Err(e) => return Box::new(std::iter::once(Err(e))), }; Box::new(std::iter::from_fn(move || { Python::attach(|py| -> Option)>> { match iter.call_method0(py, "__next__") { Ok(obj) => { if obj.is_none(py) { None } else { match obj.extract::<(u64, Vec)>(py) { Ok(pair) => Some(Ok(pair)), Err(e) => Some(Err(Error::from(e))), } } } Err(e) if e.is_instance_of::(py) => None, Err(e) => Some(Err(Error::from(e))), } }) })) } fn append_bytes( &self, relpath: &UrlFragment, bytes: &[u8], permissions: Option, ) -> Result { Python::attach(|py| { let pos = self.0.call_method1( py, "append_bytes", (relpath, bytes, perms_to_py_mode(permissions.as_ref())), )?; Ok(pos.extract::(py)?) }) } fn append_file( &self, relpath: &UrlFragment, f: &mut dyn Read, permissions: Option, ) -> Result { let f = py_read(f)?; Python::attach(|py| { let pos = self.0.call_method1( py, "append_file", (relpath, f, perms_to_py_mode(permissions.as_ref())), )?; Ok(pos.extract::(py)?) }) } fn readlink(&self, relpath: &UrlFragment) -> Result { Python::attach(|py| { Ok(self .0 .call_method1(py, "readlink", (relpath,))? .extract::(py)?) }) } fn hardlink(&self, relpath: &UrlFragment, new_relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0 .call_method1(py, "hardlink", (relpath, new_relpath))?; Ok(()) }) } fn symlink(&self, relpath: &UrlFragment, new_relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "symlink", (relpath, new_relpath))?; Ok(()) }) } fn iter_files_recursive(&self) -> Box>> { let iter = Python::attach(|py| -> Result> { let raw = self.0.call_method0(py, "iter_files_recursive")?; let it = raw.bind(py).try_iter().map_err(Error::from)?; Ok(it.unbind().into_any()) }); let iter = match iter { Ok(i) => i, Err(e) => return Box::new(std::iter::once(Err(e))), }; Box::new(std::iter::from_fn(move || { Python::attach(|py| -> Option> { match iter.call_method0(py, "__next__") { Ok(obj) => { if obj.is_none(py) { None } else { Some(obj.extract::(py).map_err(Error::from)) } } Err(e) if e.is_instance_of::(py) => None, Err(e) => Some(Err(Error::from(e))), } }) })) } fn open_write_stream( &self, relpath: &UrlFragment, permissions: Option, ) -> Result> { Python::attach(|py| { let obj = self.0.call_method1( py, "open_write_stream", (relpath, perms_to_py_mode(permissions.as_ref())), )?; let file = PyWriteStream(obj); Ok(Box::new(file) as Box) }) } fn delete_tree(&self, relpath: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "delete_tree", (relpath,))?; Ok(()) }) } fn r#move(&self, src: &UrlFragment, dst: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "move", (src, dst))?; Ok(()) }) } fn copy_tree(&self, src: &UrlFragment, dst: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "copy_tree", (src, dst))?; Ok(()) }) } fn copy_tree_to_transport(&self, _to_transport: &dyn Transport) -> Result<()> { // TODO(jelmer): bridge copy_tree_to_transport across the Py↔Rust // boundary. The Python side expects a Transport object, but we'd // need to obtain its underlying Py wrapper. For now signal the // caller that this transport can't perform the operation. Err(Error::TransportNotPossible) } fn copy_to( &self, _relpaths: &[&UrlFragment], _to_transport: &dyn Transport, _permissions: Option, ) -> Result { // TODO(jelmer): same blocker as copy_tree_to_transport — the // destination transport isn't readily expressible as a Py object // from inside the Rust adapter. Err(Error::TransportNotPossible) } fn can_roundtrip_unix_modebits(&self) -> bool { // Python convention names this `_can_roundtrip_unix_modebits` (with // a leading underscore) on every transport class; the unprefixed // form does not exist on the Python side. Default to false if the // wrapped object doesn't expose either spelling. Python::attach(|py| { self.0 .call_method0(py, "_can_roundtrip_unix_modebits") .ok() .and_then(|obj| obj.extract::(py).ok()) .unwrap_or(false) }) } fn local_abspath(&self, relpath: &UrlFragment) -> Result { Python::attach(|py| { let obj = self.0.call_method1(py, "local_abspath", (relpath,))?; Ok(obj.extract::(py)?) }) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { // Python list_dir may return a list, tuple, or iterator. Coerce via // try_iter so __next__ always works. let iter = Python::attach(|py| -> Result> { let raw = self.0.call_method1(py, "list_dir", (relpath,))?; let it = raw.bind(py).try_iter().map_err(Error::from)?; Ok(it.unbind().into_any()) }); let iter = match iter { Ok(i) => i, Err(e) => return Box::new(std::iter::once(Err(e))), }; Box::new(std::iter::from_fn(move || { Python::attach(|py| -> Option> { match iter.call_method0(py, "__next__") { Ok(obj) => { if obj.is_none(py) { None } else { Some(obj.extract::(py).map_err(|e| e.into())) } } Err(e) if e.is_instance_of::(py) => None, Err(e) => Some(Err(e.into())), } }) })) } fn listable(&self) -> bool { Python::attach(|py| { self.0 .call_method0(py, "listable") .ok() .and_then(|obj| obj.extract::(py).ok()) .unwrap_or(false) }) } fn lock_write(&self, relpath: &UrlFragment) -> Result> { Python::attach(|py| { let obj = self.0.call_method1(py, "lock_write", (relpath,))?; let file: Box = Box::new(PyLock(obj)); Ok(file) }) } fn lock_read(&self, relpath: &UrlFragment) -> Result> { Python::attach(|py| { let obj = self.0.call_method1(py, "lock_read", (relpath,))?; let file: Box = Box::new(PyLock(obj)); Ok(file) }) } fn get_smart_medium(&self) -> Result> { Python::attach(|py| { let obj = match self.0.call_method0(py, "get_smart_medium") { Ok(o) => o, Err(e) if e.is_instance_of::(py) => { return Err(Error::NoSmartMedium); } Err(e) => return Err(Error::Io(std::io::Error::other(e.to_string()))), }; if obj.is_none(py) { return Err(Error::NoSmartMedium); } let medium = PySmartMedium(obj); Ok(Box::new(medium) as Box) }) } fn copy(&self, src: &UrlFragment, dst: &UrlFragment) -> Result<()> { Python::attach(|py| { self.0.call_method1(py, "copy", (src, dst))?; Ok(()) }) } } dromedary-0.1.1/src/readonly.rs000064400000000000000000000255001046102023000145650ustar 00000000000000//! Readonly Transport decorator, ported from dromedary/readonly.py. //! //! Wraps any Transport and rejects every mutation with TransportNotPossible, //! forwarding read-only operations unchanged. use crate::lock::Lock; use crate::{Error, ReadStream, Result, SmartMedium, Stat, Transport, UrlFragment, WriteStream}; use std::collections::HashMap; use std::fs::Permissions; use url::Url; pub struct ReadonlyTransport { decorated: Box, base: Url, } impl ReadonlyTransport { const PREFIX: &'static str = "readonly+"; pub fn new(decorated: Box) -> Self { let inner_base = decorated.base(); let base = Url::parse(&format!("{}{}", Self::PREFIX, inner_base)).unwrap_or(inner_base.clone()); Self { decorated, base } } } impl std::fmt::Debug for ReadonlyTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "ReadonlyTransport({})", self.base) } } fn not_possible() -> Error { Error::TransportNotPossible } impl Transport for ReadonlyTransport { fn external_url(&self) -> Result { self.decorated.external_url() } fn can_roundtrip_unix_modebits(&self) -> bool { self.decorated.can_roundtrip_unix_modebits() } fn base(&self) -> Url { self.base.clone() } fn is_readonly(&self) -> bool { true } crate::fwd_listable!(decorated); fn get(&self, relpath: &UrlFragment) -> Result> { self.decorated.get(relpath) } fn has(&self, relpath: &UrlFragment) -> Result { self.decorated.has(relpath) } fn stat(&self, relpath: &UrlFragment) -> Result { self.decorated.stat(relpath) } fn clone(&self, offset: Option<&UrlFragment>) -> Result> { // NB: once a trait upcast to Send+Sync decorator stacking is needed we // will revisit; for now a cloned ReadonlyTransport is Send+Sync but // the cloned inner handle is plain Box, so we return // the inner transport's clone directly and let callers re-wrap. self.decorated.clone(offset) } fn abspath(&self, relpath: &UrlFragment) -> Result { self.decorated.abspath(relpath) } fn relpath(&self, abspath: &Url) -> Result { self.decorated.relpath(abspath) } fn put_file( &self, _relpath: &UrlFragment, _f: &mut dyn std::io::Read, _permissions: Option, ) -> Result { Err(not_possible()) } fn mkdir(&self, _relpath: &UrlFragment, _permissions: Option) -> Result<()> { Err(not_possible()) } fn delete(&self, _relpath: &UrlFragment) -> Result<()> { Err(not_possible()) } fn rmdir(&self, _relpath: &UrlFragment) -> Result<()> { Err(not_possible()) } fn rename(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(not_possible()) } fn set_segment_parameter(&mut self, key: &str, value: Option<&str>) -> Result<()> { // Forwarding a &mut call to the inner transport is awkward with a // Box; defer to the inner via the mutable reference // we hold. self.decorated.set_segment_parameter(key, value) } fn get_segment_parameters(&self) -> Result> { self.decorated.get_segment_parameters() } fn append_file( &self, _relpath: &UrlFragment, _f: &mut dyn std::io::Read, _permissions: Option, ) -> Result { Err(not_possible()) } fn readlink(&self, relpath: &UrlFragment) -> Result { self.decorated.readlink(relpath) } fn hardlink(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(not_possible()) } fn symlink(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(not_possible()) } fn iter_files_recursive(&self) -> Box>> { self.decorated.iter_files_recursive() } fn open_write_stream( &self, _relpath: &UrlFragment, _permissions: Option, ) -> Result> { Err(not_possible()) } fn delete_tree(&self, _relpath: &UrlFragment) -> Result<()> { Err(not_possible()) } fn r#move(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(not_possible()) } fn list_dir(&self, relpath: &UrlFragment) -> Box>> { self.decorated.list_dir(relpath) } fn lock_read(&self, relpath: &UrlFragment) -> Result> { self.decorated.lock_read(relpath) } fn lock_write(&self, _relpath: &UrlFragment) -> Result> { Err(not_possible()) } fn local_abspath(&self, relpath: &UrlFragment) -> Result { self.decorated.local_abspath(relpath) } fn get_smart_medium(&self) -> Result> { Err(Error::NoSmartMedium) } fn copy(&self, _rel_from: &UrlFragment, _rel_to: &UrlFragment) -> Result<()> { Err(not_possible()) } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn ro() -> ReadonlyTransport { let mem = MemoryTransport::new("memory:///").unwrap(); mem.put_bytes("hello", b"world", None).unwrap(); ReadonlyTransport::new(Box::new(mem)) } #[test] fn reads_pass_through() { let t = ro(); assert_eq!(t.get_bytes("hello").unwrap(), b"world"); assert_eq!(t.has("hello").unwrap(), true); assert_eq!(t.has("missing").unwrap(), false); } #[test] fn is_readonly_returns_true() { assert!(ro().is_readonly()); } #[test] fn put_bytes_rejected() { match ro().put_bytes("x", b"y", None) { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn mkdir_rejected() { match ro().mkdir("d", None) { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn delete_rejected() { match ro().delete("hello") { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn rename_rejected() { match ro().rename("hello", "world") { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn lock_read_passes_but_lock_write_rejected() { let t = ro(); let _l = t.lock_read("hello").ok().expect("read lock"); match t.lock_write("hello") { Err(Error::TransportNotPossible) => {} Err(other) => panic!("expected TransportNotPossible, got {:?}", other), Ok(_) => panic!("expected TransportNotPossible, got Ok"), } } #[test] fn base_has_readonly_prefix() { let t = ro(); assert!(t.base().as_str().starts_with("readonly+")); } fn expect_not_possible(r: Result, label: &str) { match r { Err(Error::TransportNotPossible) => {} Err(other) => panic!("{}: expected TransportNotPossible, got {:?}", label, other), Ok(ok) => panic!("{}: expected TransportNotPossible, got Ok({:?})", label, ok), } } #[test] fn rmdir_rejected() { // Seed the inner store with a directory we could otherwise remove. let mem = MemoryTransport::new("memory:///").unwrap(); mem.mkdir("d", None).unwrap(); let t = ReadonlyTransport::new(Box::new(mem)); expect_not_possible(t.rmdir("d"), "rmdir"); } #[test] fn delete_tree_rejected() { expect_not_possible(ro().delete_tree("hello"), "delete_tree"); } #[test] fn symlink_rejected() { expect_not_possible(ro().symlink("hello", "link"), "symlink"); } #[test] fn hardlink_rejected() { expect_not_possible(ro().hardlink("hello", "link"), "hardlink"); } #[test] fn append_file_rejected() { let mut cur = std::io::Cursor::new(b"more".to_vec()); expect_not_possible(ro().append_file("hello", &mut cur, None), "append_file"); } #[test] fn append_bytes_rejected() { expect_not_possible(ro().append_bytes("hello", b"more", None), "append_bytes"); } #[test] fn open_write_stream_rejected() { match ro().open_write_stream("hello", None) { Err(Error::TransportNotPossible) => {} Err(other) => panic!("expected TransportNotPossible, got {:?}", other), Ok(_) => panic!("expected TransportNotPossible, got Ok"), } } #[test] fn move_rejected() { expect_not_possible(ro().r#move("hello", "world"), "move"); } #[test] fn copy_rejected() { expect_not_possible(ro().copy("hello", "world"), "copy"); } #[test] fn stat_passes_through() { let st = ro().stat("hello").unwrap(); assert_eq!(st.size, 5); } #[test] fn list_dir_passes_through() { let mem = MemoryTransport::new("memory:///").unwrap(); mem.mkdir("d", None).unwrap(); mem.put_bytes("d/a", b"1", None).unwrap(); mem.put_bytes("d/b", b"2", None).unwrap(); let t = ReadonlyTransport::new(Box::new(mem)); let mut entries: Vec = t.list_dir("d").filter_map(|r| r.ok()).collect(); entries.sort(); assert_eq!(entries, vec!["a".to_string(), "b".to_string()]); } #[test] fn iter_files_recursive_passes_through() { let mem = MemoryTransport::new("memory:///").unwrap(); mem.put_bytes("a", b"1", None).unwrap(); mem.put_bytes("b", b"2", None).unwrap(); let t = ReadonlyTransport::new(Box::new(mem)); let mut files: Vec = t.iter_files_recursive().filter_map(|r| r.ok()).collect(); files.sort(); assert_eq!(files, vec!["a".to_string(), "b".to_string()]); } #[test] fn external_url_forwards_error() { // MemoryTransport returns InProcessTransport; readonly should forward it. match ro().external_url() { Err(Error::InProcessTransport) => {} other => panic!("expected InProcessTransport, got {:?}", other), } } #[test] fn get_missing_forwards_no_such_file() { match ro().get_bytes("nope") { Err(Error::NoSuchFile(_)) => {} other => panic!("expected NoSuchFile, got {:?}", other), } } } dromedary-0.1.1/src/readv.rs000064400000000000000000000216361046102023000140570ustar 00000000000000use std::collections::HashMap; use std::collections::VecDeque; use std::io::{Read, Seek, SeekFrom}; pub struct OverlappingRange { last_end: usize, start: usize, } impl std::fmt::Display for OverlappingRange { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!( f, "Overlapping range not allowed: last range ended at {}, new one starts at {}", self.last_end, self.start ) } } /// Yield coalesced offsets. /// /// With a long list of neighboring requests, combine them /// into a single large request, while retaining the original /// offsets. /// Turns [(15, 10), (25, 10)] => [(15, 20, [(0, 10), (10, 10)])] /// Note that overlapping requests are not permitted. (So [(15, 10), (20, /// 10)] will raise a ValueError.) This is because the data we access never /// overlaps, and it allows callers to trust that we only need any byte of /// data for 1 request (so nothing needs to be buffered to fulfill a second /// request.) /// /// :param offsets: A list of (start, length) pairs /// :param limit: Only combine a maximum of this many pairs Some transports /// penalize multiple reads more than others, and sometimes it is /// better to return early. /// 0 means no limit /// :param fudge_factor: All transports have some level of 'it is /// better to read some more data and throw it away rather /// than seek', so collapse if we are 'close enough' /// :param max_size: Create coalesced offsets no bigger than this size. /// When a single offset is bigger than 'max_size', it will keep /// its size and be alone in the coalesced offset. /// 0 means no maximum size. /// :return: return a list of _CoalescedOffset objects, which have members /// for where to start, how much to read, and how to split those chunks /// back up pub fn coalesce_offsets( offsets: &[(usize, usize)], limit: Option, fudge_factor: Option, max_size: Option, ) -> std::result::Result)>, OverlappingRange> { let mut offsets = offsets.to_vec(); offsets.sort(); struct CoalescedOffset { start: usize, length: usize, ranges: Vec<(usize, usize)>, } if offsets.is_empty() { return Ok(vec![]); } let mut cur = CoalescedOffset { start: offsets[0].0, length: offsets[0].1, ranges: vec![(0, offsets[0].1)], }; let mut last_end = cur.start + cur.length; let mut coalesced_offsets = Vec::new(); let fudge_factor = fudge_factor.unwrap_or(0); // unlimited, but we actually take this to mean 100MB buffer limit let max_size = max_size.unwrap_or(100 * 1024 * 1024); for (start, size) in &offsets[1..] { let end = start + size; if *start <= last_end + fudge_factor && *start >= cur.start && (limit.is_none() || cur.ranges.len() < limit.unwrap()) && (end - cur.start <= max_size) { if *start < last_end { return Err(OverlappingRange { last_end, start: *start, }); } cur.length = end - cur.start; cur.ranges.push((start - cur.start, *size)); } else { coalesced_offsets.push((cur.start, cur.length, cur.ranges)); cur = CoalescedOffset { start: *start, length: *size, ranges: vec![(0, *size)], }; } last_end = end; } coalesced_offsets.push((cur.start, cur.length, cur.ranges)); Ok(coalesced_offsets) } struct ReadvIter { fp: T, offsets: VecDeque<(usize, usize)>, coalesced: VecDeque<(usize, usize, Vec<(usize, usize)>)>, data_map: HashMap<(usize, usize), Vec>, } impl ReadvIter { fn new( fp: T, offsets: Vec<(usize, usize)>, max_readv_combine: usize, bytes_to_read_before_seek: usize, ) -> std::io::Result { // Turn list of offsets into a stack let coalesced = coalesce_offsets( &offsets, Some(max_readv_combine), Some(bytes_to_read_before_seek), None, ) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?; Ok(Self { fp, offsets: VecDeque::from(offsets), coalesced: coalesced.into_iter().collect(), data_map: std::collections::HashMap::new(), }) } fn read_more(&mut self) -> Result { // Cache the results, but only until they have been fulfilled if let Some((start, length, ranges)) = self.coalesced.pop_front() { self.fp .seek(SeekFrom::Start(start as u64)) .map_err(|e| (e, start, length, 0))?; let mut data = vec![0; length]; self.fp .read_exact(&mut data) .map_err(|e| (e, start, length, 0))?; for (suboffset, subsize) in ranges { self.data_map.insert( (start + suboffset, subsize), data[suboffset..suboffset + subsize].to_vec(), ); } Ok(true) } else { Ok(false) } } } impl Iterator for ReadvIter { type Item = Result<(usize, Vec), (std::io::Error, usize, usize, usize)>; fn next(&mut self) -> Option { if let Some(key) = self.offsets.pop_front() { loop { if let Some(data) = self.data_map.remove(&key) { break Some(Ok((key.0, data))); } else { match self.read_more() { Ok(true) => continue, Ok(false) => break None, Err(e) => break Some(Err(e)), } } } } else { None } } } /// An implementation of readv that uses fp.seek and fp.read. /// /// This uses _coalesce_offsets to issue larger reads and fewer seeks. /// /// :param fp: A file-like object that supports seek() and read(size). /// Note that implementations are allowed to call .close() on this file /// handle, so don't trust that you can use it for other work. /// :param offsets: A list of offsets to be read from the given file. /// :return: yield (pos, data) tuples for each request pub fn seek_and_read( fp: T, offsets: Vec<(usize, usize)>, max_readv_combine: usize, bytes_to_read_before_seek: usize, ) -> std::io::Result< impl Iterator), (std::io::Error, usize, usize, usize)>>, > { ReadvIter::new(fp, offsets, max_readv_combine, bytes_to_read_before_seek) } pub fn sort_expand_and_combine( offsets: Vec<(u64, usize)>, upper_limit: Option, recommended_page_size: usize, ) -> Vec<(u64, usize)> { // Sort the offsets by start address. let mut sorted_offsets = offsets.to_vec(); sorted_offsets.sort_unstable_by_key(|&(offset, _)| offset); // Short circuit empty requests. if sorted_offsets.is_empty() { return Vec::new(); } // Expand the offsets by page size at either end. let maximum_expansion = recommended_page_size; let mut new_offsets = Vec::with_capacity(sorted_offsets.len()); for (offset, length) in sorted_offsets { let expansion = maximum_expansion.saturating_sub(length); let reduction = expansion / 2; let new_offset = offset.saturating_sub(reduction as u64); let new_length = length + expansion; let new_length = if let Some(upper_limit) = upper_limit { let new_end = new_offset.saturating_add(new_length as u64); let new_length = std::cmp::min(upper_limit, new_end) - new_offset; std::cmp::max(0, new_length as isize) as usize } else { new_length }; if new_length > 0 { new_offsets.push((new_offset, new_length)); } } // Combine the expanded offsets. let mut result = Vec::with_capacity(new_offsets.len()); if let Some((mut current_offset, mut current_length)) = new_offsets.first().copied() { let mut current_finish = current_offset + current_length as u64; for (offset, length) in new_offsets.iter().skip(1) { let finish = offset + *length as u64; if *offset > current_finish { result.push((current_offset, current_length)); current_offset = *offset; current_length = *length; current_finish = finish; } else if finish > current_finish { current_finish = finish; current_length = (current_finish - current_offset) as usize; } } result.push((current_offset, current_length)); } result } dromedary-0.1.1/src/unlistable.rs000064400000000000000000000072611046102023000151160ustar 00000000000000//! Unlistable Transport decorator, ported from dromedary/unlistable.py. //! //! A transport that disables directory listing, to simulate HTTP cheaply //! in tests. `listable()` returns false; `list_dir` and //! `iter_files_recursive` both yield a single TransportNotPossible error. use crate::{Error, Result, Transport, UrlFragment}; use url::Url; pub struct UnlistableTransport { inner: Box, base: Url, } impl UnlistableTransport { pub const PREFIX: &'static str = "unlistable+"; pub fn new(inner: Box) -> Self { let base = crate::decorator::prefixed_base(Self::PREFIX, inner.as_ref()); Self { inner, base } } } impl std::fmt::Debug for UnlistableTransport { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "UnlistableTransport({})", self.base) } } impl Transport for UnlistableTransport { crate::fwd_external_url!(inner); crate::fwd_can_roundtrip_unix_modebits!(inner); crate::fwd_is_readonly!(inner); crate::fwd_get!(inner); crate::fwd_has!(inner); crate::fwd_stat!(inner); crate::fwd_clone!(inner); crate::fwd_abspath!(inner); crate::fwd_relpath!(inner); crate::fwd_put_file!(inner); crate::fwd_mkdir!(inner); crate::fwd_delete!(inner); crate::fwd_rmdir!(inner); crate::fwd_rename!(inner); crate::fwd_set_segment_parameter!(inner); crate::fwd_get_segment_parameters!(inner); crate::fwd_append_file!(inner); crate::fwd_readlink!(inner); crate::fwd_hardlink!(inner); crate::fwd_symlink!(inner); crate::fwd_open_write_stream!(inner); crate::fwd_delete_tree!(inner); crate::fwd_move!(inner); crate::fwd_lock_read!(inner); crate::fwd_lock_write!(inner); crate::fwd_local_abspath!(inner); crate::fwd_get_smart_medium!(inner); crate::fwd_copy!(inner); fn base(&self) -> Url { self.base.clone() } fn listable(&self) -> bool { false } fn list_dir(&self, _relpath: &UrlFragment) -> Box>> { Box::new(std::iter::once(Err(Error::TransportNotPossible))) } fn iter_files_recursive(&self) -> Box>> { Box::new(std::iter::once(Err(Error::TransportNotPossible))) } } #[cfg(test)] mod tests { use super::*; use crate::memory::MemoryTransport; fn wrap() -> UnlistableTransport { let mem = MemoryTransport::new("memory:///").unwrap(); mem.put_bytes("a", b"1", None).unwrap(); mem.put_bytes("b", b"2", None).unwrap(); UnlistableTransport::new(Box::new(mem)) } #[test] fn base_prefix() { assert!(wrap().base().as_str().starts_with("unlistable+")); } #[test] fn listable_returns_false() { assert_eq!(wrap().listable(), false); } #[test] fn list_dir_yields_not_possible() { let t = wrap(); let results: Vec> = t.list_dir(".").collect(); assert_eq!(results.len(), 1); match &results[0] { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn iter_files_recursive_yields_not_possible() { let t = wrap(); let results: Vec> = t.iter_files_recursive().collect(); assert_eq!(results.len(), 1); match &results[0] { Err(Error::TransportNotPossible) => {} other => panic!("expected TransportNotPossible, got {:?}", other), } } #[test] fn reads_pass_through() { let t = wrap(); assert_eq!(t.get_bytes("a").unwrap(), b"1"); } } dromedary-0.1.1/src/urlutils.rs000064400000000000000000000774021046102023000146430ustar 00000000000000use lazy_static::lazy_static; use regex::Regex; use std::collections::HashMap; use std::path::{Path, PathBuf}; lazy_static! { static ref URL_SCHEME_RE: Regex = Regex::new(r"^(?P[^:/]{2,}):(//)?(?P.*)$").unwrap(); static ref URL_HEX_ESCAPES_RE: Regex = Regex::new(r"(%[0-9a-fA-F]{2})").unwrap(); } #[derive(Debug)] pub enum Error { AboveRoot(String, String), SubsegmentMissesEquals(String), UnsafeCharacters(char), IoError(std::io::Error), SegmentParameterKeyContainsEquals(String, String), SegmentParameterContainsComma(String, Vec), NotLocalUrl(String), InvalidUNCUrl(String), UrlNotAscii(String), InvalidWin32LocalUrl(String), InvalidWin32Path(String), UrlTooShort(String), PathNotChild(String, String), InvalidUrlPort(String, String), } type Result = std::result::Result; /// Split a URL into its parent directory and a child directory. /// /// Args: /// url: A relative or absolute URL /// exclude_trailing_slash: Strip off a final '/' if it is part /// of the path (but not if it is part of the protocol specification) /// /// Returns: (parent_url, child_dir). child_dir may be the empty string if /// we're at the root. pub fn split(url: &str, exclude_trailing_slash: bool) -> (String, String) { let (scheme_loc, first_path_slash) = find_scheme_and_separator(url); if first_path_slash.is_none() { // We have either a relative path, or no separating slash if scheme_loc.is_none() { // Relative path let mut url = url; if exclude_trailing_slash && url.ends_with('/') { url = &url[..url.len() - 1]; } let split = url.rsplit_once('/').map(|(head, tail)| { if head.is_empty() { ("/", tail) } else { (head, tail) } }); match split { None => return (String::new(), url.to_string()), Some((head, tail)) => return (head.to_string(), tail.to_string()), } } else { // Scheme with no path return (url.to_string(), String::new()); } } // We have a fully defined path let url_base = &url[..first_path_slash.unwrap()]; // http://host, file:// let mut path = &url[first_path_slash.unwrap()..]; // /file/foo // TODO(windows): the original breezy code rebinds `url_base`/`path` here // via `_extract_drive_letter` so that `file:///C:/foo` splits as // `file:///C:` + `/foo` rather than `file://` + `/C:/foo`. The direct port // below shadowed but never used the rebinding, so on Windows `split` for // drive-letter URLs is currently wrong. See `win32::extract_drive_letter`. #[cfg(target_os = "windows")] { // Reference the symbol so the cfg-gated branch compiles; no-op for now. let _ = win32::extract_drive_letter; } if exclude_trailing_slash && path.len() > 1 && path.ends_with('/') { path = &path[..path.len() - 1]; } let split = path.rsplit_once('/').map(|(head, tail)| { if head.is_empty() { ("/", tail) } else { (head, tail) } }); match split { None => (url_base.to_string(), path.to_string()), Some((head, tail)) => (url_base.to_string() + head, tail.to_string()), } } /// Find the scheme separator (://) and the first path separator /// /// This is just a helper functions for other path utilities. /// It could probably be replaced by urlparse pub fn find_scheme_and_separator(url: &str) -> (Option, Option) { if let Some(m) = URL_SCHEME_RE.captures(url) { let scheme = m.name("scheme").unwrap().as_str(); let path = m.name("path").unwrap().as_str(); // Find the path separating slash // (first slash after the ://) if let Some(first_path_slash) = path.find('/') { ( Some(scheme.len()), Some(first_path_slash + m.name("path").unwrap().start()), ) } else { (Some(scheme.len()), None) } } else { (None, None) } } pub fn is_url(url: &str) -> bool { // Tests whether a URL is in actual fact a URL. URL_SCHEME_RE.is_match(url) } /// Strip trailing slash, except for root paths. /// /// The definition of 'root path' is platform-dependent. /// This assumes that all URLs are valid netloc urls, such that they /// form: /// scheme://host/path /// It searches for ://, and then refuses to remove the next '/'. /// It can also handle relative paths /// Examples: /// path/to/foo => path/to/foo /// path/to/foo/ => path/to/foo /// http://host/path/ => http://host/path /// http://host/path => http://host/path /// http://host/ => http://host/ /// file:/// => file:/// /// file:///foo/ => file:///foo /// # This is unique on win32 platforms, and is the only URL /// # format which does it differently. /// file:///c|/ => file:///c:/ pub fn strip_trailing_slash(url: &str) -> &str { if !url.ends_with('/') { // Nothing to do return url; } // TODO(windows): `win32::strip_local_trailing_slash` returns `String`, // so we can't return it from this `&str`-returning function directly. // The Python original returned a new string here; porting that requires // changing the signature to `Cow<'_, str>`. For now Windows callers get // the same generic handling as Unix, which is incorrect for drive-letter // file:/// URLs but compiles. let (scheme_loc, first_path_slash) = find_scheme_and_separator(url); if scheme_loc.is_none() { // This is a relative path, as it has no scheme // so just chop off the last character &url[..url.len() - 1] } else if first_path_slash.is_none() || first_path_slash.unwrap() == url.len() - 1 { // Don't chop off anything if the only slash is the path // separating slash url } else { &url[..url.len() - 1] } } /// Join URL path segments to a URL path segment. /// /// This is somewhat like osutils.joinpath, but intended for URLs. /// /// XXX: this duplicates some normalisation logic, and also duplicates a lot of /// path handling logic that already exists in some Transport implementations. /// We really should try to have exactly one place in the code base responsible /// for combining paths of URLs. pub fn joinpath(base: &str, args: &[&str]) -> Result { let mut path = base.split('/').collect::>(); if path.len() > 1 && path[path.len() - 1].is_empty() { // If the path ends in a trailing /, remove it. path.pop(); } for arg in args { if arg.starts_with('/') { path = vec![]; } for chunk in arg.split('/') { if chunk == "." { continue; } else if chunk == ".." { if path == [""] { return Err(Error::AboveRoot(base.to_string(), args.join("/"))); } path.pop(); } else { path.push(chunk); } } } Ok(if path == [""] { "/".to_string() } else { path.join("/") }) } /// Return the last component of a URL. /// /// Args: /// url The URL in question /// exclude_trailing_slash: If the url looks like "path/to/foo/", /// ignore the final slash and return 'foo' rather than '' /// Returns: /// Just the final component of the URL. This can return '' /// if you don't exclude_trailing_slash, or if you are at the /// root of the URL. pub fn basename(url: &str, exclude_trailing_slash: bool) -> String { split(url, exclude_trailing_slash).1 } /// Return the parent directory of the given path. /// /// Args: /// url: Relative or absolute URL /// exclude_trailing_slash: Remove a final slash (treat http://host/foo/ as http://host/foo, but /// http://host/ stays http://host/) /// /// Returns: Everything in the URL except the last path chunk // jam 20060502: This was named dirname to be consistent // with the os functions, but maybe "parent" would be better pub fn dirname(url: &str, exclude_trailing_slash: bool) -> String { split(url, exclude_trailing_slash).0 } /// Create a URL by joining sections. /// /// This will normalize '..', assuming that paths are absolute /// (it assumes no symlinks in either path) /// /// If any of *args is an absolute URL, it will be treated correctly. /// Example: /// join('http://foo', 'http://bar') => 'http://bar' /// join('http://foo', 'bar') => 'http://foo/bar' /// join('http://foo', 'bar', '../baz') => 'http://foo/baz' pub fn join<'a>(mut base: &'a str, args: &[&'a str]) -> Result { if args.is_empty() { return Ok(base.to_string()); } let (scheme_end, path_start) = find_scheme_and_separator(base); let mut path_start = if scheme_end.is_none() && path_start.is_none() { 0 } else if path_start.is_none() { base.len() } else { path_start.unwrap() }; let mut path = base[path_start..].to_string(); for arg in args { let (arg_scheme_end, arg_path_start) = find_scheme_and_separator(arg); let arg_path_start = if arg_scheme_end.is_none() && arg_path_start.is_none() { 0 } else if arg_path_start.is_none() { arg.len() } else { arg_path_start.unwrap() }; if arg_scheme_end.is_some() { base = arg; path = arg[arg_path_start..].to_string(); path_start = arg_path_start; } else { path = joinpath(path.as_str(), vec![*arg].as_slice())?; } } Ok(base[..path_start].to_string() + &path) } /// Split the subsegment of the last segment of a URL. /// ///Args: /// url: A relative or absolute URL ///Returns: (url, subsegments) pub fn split_segment_parameters_raw(url: &str) -> (&str, Vec<&str>) { // GZ 2011-11-18: Dodgy removing the terminal slash like this, function // operates on urls not url+segments, and Transport classes // should not be blindly adding slashes in the first place. let lurl = strip_trailing_slash(url); let segment_start = lurl.rfind('/').map_or_else(|| 0, |i| i + 1); if !lurl[segment_start..].contains(',') { return (url, vec![]); } let mut iter = lurl[segment_start..].split(','); let first = iter.next().unwrap(); ( &lurl[..segment_start + first.len()], iter.map(|s| s.trim()).collect(), ) } /// Split the segment parameters of the last segment of a URL. /// /// Args: /// url: A relative or absolute URL /// Returns: (url, segment_parameters) pub fn split_segment_parameters( url: &str, ) -> Result<(&str, std::collections::HashMap<&str, &str>)> { let (base_url, subsegments) = split_segment_parameters_raw(url); let parameters = subsegments .iter() .map(|subsegment| { subsegment .split_once('=') .ok_or_else(|| Error::SubsegmentMissesEquals(subsegment.to_string())) .map(|(key, value)| (key.trim(), value.trim())) }) .collect::>>()?; Ok((base_url, parameters)) } /// Strip the segment parameters from a URL. /// /// Args: /// url: A relative or absolute URL /// Returns: url pub fn strip_segment_parameters(url: &str) -> &str { split_segment_parameters_raw(url).0 } /// Create a new URL by adding subsegments to an existing one. /// /// This adds the specified subsegments to the last path in the specified /// base URL. The subsegments should be bytestrings. /// /// Note: You probably want to use join_segment_parameters instead. pub fn join_segment_parameters_raw(base: &str, subsegments: &[&str]) -> Result { if subsegments.is_empty() { return Ok(base.to_string()); } for subsegment in subsegments { if subsegment.contains(',') { return Err(Error::SegmentParameterContainsComma( base.to_string(), subsegments.iter().map(|s| s.to_string()).collect(), )); } } Ok(format!("{},{}", base, subsegments.join(","))) } /// Create a new URL by adding segment parameters to an existing one. /// /// The parameters of the last segment in the URL will be updated; if a /// parameter with the same key already exists it will be overwritten. /// /// Args: /// url: A URL, as string /// parameters: Dictionary of parameters, keys and values as bytestrings pub fn join_segment_parameters(url: &str, parameters: &HashMap<&str, &str>) -> Result { let (base, existing_parameters) = split_segment_parameters(url)?; let mut new_parameters = existing_parameters.clone(); for (key, value) in parameters { if key.contains('=') { return Err(Error::SegmentParameterKeyContainsEquals( url.to_string(), key.to_string(), )); } new_parameters.insert(key, value); } let mut items: Vec<_> = new_parameters.iter().collect(); items.sort_by(|a, b| a.0.cmp(b.0)); let sorted_parameters: Vec<_> = items .iter() .map(|(key, value)| format!("{}={}", key, value)) .collect(); join_segment_parameters_raw( base, &sorted_parameters .iter() .map(|s| s.as_str()) .collect::>(), ) } /// Return a path to other from base. /// /// If other is unrelated to base, return other. Else return a relative path. /// This assumes no symlinks as part of the url. pub fn relative_url(base: &str, other: &str) -> String { let (_, base_first_slash) = find_scheme_and_separator(base); if base_first_slash.is_none() { return other.to_string(); } let (_, other_first_slash) = find_scheme_and_separator(other); if other_first_slash.is_none() { return other.to_string(); } // this takes care of differing schemes or hosts let base_scheme = &base[..base_first_slash.unwrap()]; let other_scheme = &other[..other_first_slash.unwrap()]; if base_scheme != other_scheme { return other.to_string(); } #[cfg(target_os = "windows")] if base_scheme == "file://" { let base_drive = &base[base_first_slash.unwrap() + 1..base_first_slash.unwrap() + 3]; let other_drive = &other[other_first_slash.unwrap() + 1..other_first_slash.unwrap() + 3]; if base_drive != other_drive { return other.to_string(); } } let mut base_path = &base[base_first_slash.unwrap() + 1..]; let other_path = &other[other_first_slash.unwrap() + 1..]; if base_path.ends_with('/') { base_path = &base_path[..base_path.len() - 1]; } let mut base_sections: Vec<_> = base_path.split('/').collect(); let mut other_sections: Vec<_> = other_path.split('/').collect(); if base_sections == [""] { base_sections = Vec::new(); } if other_sections == [""] { other_sections = Vec::new(); } let mut output_sections = Vec::new(); for (b, o) in base_sections.iter().zip(other_sections.iter()) { if b != o { break; } output_sections.push(b); } let match_len = output_sections.len(); let mut output_sections: Vec<_> = base_sections[match_len..].iter().map(|_x| "..").collect(); output_sections.extend_from_slice(&other_sections[match_len..]); let ret = output_sections.join("/"); if ret.is_empty() { ".".to_string() } else { ret } } fn char_is_safe(c: char) -> bool { c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' } fn unescape_safe_chars(captures: ®ex::Captures) -> String { let hex_digits = &captures[0][1..]; let char_code = u8::from_str_radix(hex_digits, 16).unwrap(); let character = char::from(char_code); if char_is_safe(character) { character.to_string() } else { captures[0].to_uppercase() } } /// Transform a Transport-relative path to a remote absolute path. /// /// This does not handle substitution of ~ but does handle '..' and '.' /// components. /// /// # Examples /// /// use dromedary::urlutils::combine_paths; /// assert_eq!("/home/sarah/project/foo", combine_paths("/home/sarah", "project/foo")); /// assert_eq!("/etc", combine_paths("/home/sarah", "../../etc")); /// assert_eq!("/etc", combine_paths("/home/sarah", "/etc")); /// /// # Arguments /// /// * `base_path` - base path /// * `relpath` - relative path /// /// # Returns /// /// urlencoded string for final path. pub fn combine_paths(base_path: &str, relpath: &str) -> String { let relpath = URL_HEX_ESCAPES_RE .replace_all(relpath, unescape_safe_chars) .to_string(); let mut base_parts: Vec<&str> = if relpath.starts_with('/') { vec![] } else { base_path.split('/').collect() }; if base_parts.last() == Some(&"") { base_parts.pop(); } for p in relpath.split('/') { match p { ".." => { if let Some(last) = base_parts.last() { if !last.is_empty() { base_parts.pop(); } } } "." | "" => (), _ => base_parts.push(p), } } let mut path = base_parts.join("/"); if !path.starts_with('/') { path.insert(0, '/'); } path } /// Make sure that a path string is in fully normalized URL form. /// /// This handles URLs which have unicode characters, spaces, /// special characters, etc. /// /// It has two basic modes of operation, depending on whether the /// supplied string starts with a url specifier (scheme://) or not. /// If it does not have a specifier it is considered a local path, /// and will be converted into a file:/// url. Non-ascii characters /// will be encoded using utf-8. /// If it does have a url specifier, it will be treated as a "hybrid" /// URL. Basically, a URL that should have URL special characters already /// escaped (like +?&# etc), but may have unicode characters, etc /// which would not be valid in a real URL. /// /// Args: /// url: Either a hybrid URL or a local path /// Returns: A normalized URL which only includes 7-bit ASCII characters. pub fn normalize_url(url: &str) -> Result { let (scheme_end, path_start) = find_scheme_and_separator(url); if scheme_end.is_none() { local_path_to_url(url).map_err(Error::IoError) } else { let prefix = &url[..path_start.unwrap()]; let path = &url[path_start.unwrap()..]; // These characters should not be escaped const URL_SAFE_CHARACTERS: &[u8] = b"_.-!~*'()/;?:@&=+$,%#"; let path = path .as_bytes() .iter() .map(|c| { if !c.is_ascii_alphanumeric() && !URL_SAFE_CHARACTERS.contains(c) { format!("%{:02X}", c) } else { (*c as char).to_string() } }) .collect::(); let path = URL_HEX_ESCAPES_RE.replace_all(path.as_str(), unescape_safe_chars); Ok(prefix.to_string() + path.as_ref()) } } pub fn escape(relpath: &[u8], safe: Option<&str>) -> String { let mut result = String::new(); let safe = safe.unwrap_or("/~").as_bytes(); for b in relpath { if char_is_safe(char::from(*b)) || safe.contains(b) { result.push(char::from(*b)); } else { result.push_str(&format!("%{:02X}", *b)); } } result } pub fn unescape(url: &str) -> Result { use percent_encoding::percent_decode_str; if !url.is_ascii() { return Err(Error::UrlNotAscii(url.to_string())); } Ok(percent_decode_str(url) .decode_utf8() .map(|s| s.to_string()) .unwrap_or_else(|_| url.to_string())) } pub mod win32 { use std::path::{Path, PathBuf}; /// Convert a local path like ./foo into a URL like file:///C:/path/to/foo /// /// This also handles transforming escaping unicode characters, etc. pub fn local_path_to_url>(path: P) -> std::io::Result { if path.as_ref().as_os_str() == "/" { return Ok("file:///".to_string()); } let win32_path = crate::osutils::path::win32::abspath(path.as_ref())?; let win32_path = win32_path.as_path().to_str().unwrap(); if win32_path.starts_with("//") { Ok(format!( "file:{}", super::escape(win32_path.as_bytes(), Some("/~")) )) } else { let drive = win32_path.chars().next().unwrap().to_ascii_uppercase(); Ok(format!( "file:///{}:{}", drive, super::escape(win32_path[2..].as_bytes(), Some("/~")) )) } } /// Convert a url like file:///C:/path/to/foo into C:/path/to/foo pub fn local_path_from_url(url: &str) -> super::Result { if !url.starts_with("file://") { return Err(super::Error::NotLocalUrl(url.to_string())); } let url = super::strip_segment_parameters(url); let win32_url = &url[5..]; if !win32_url.starts_with("///") { if win32_url.len() < 3 || win32_url.chars().nth(2).unwrap() == '/' || "|:".contains(win32_url.chars().nth(3).unwrap()) { return Err(super::Error::InvalidUNCUrl(url.to_string())); } return Ok(super::unescape(win32_url)?.into()); } // Allow empty paths so we can serve all roots if win32_url == "///" { return Ok(PathBuf::from("/")); } // Usual local path with drive letter if win32_url.len() < 6 || !("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".contains(&win32_url[3..=3])) || !("|:".contains(win32_url.chars().nth(4).unwrap())) || win32_url.chars().nth(5) != Some('/') { return Err(super::Error::InvalidWin32LocalUrl(url.to_string())); } Ok(PathBuf::from(format!( "{}:{}", win32_url[3..=3].to_uppercase(), super::unescape(&win32_url[5..])? ))) } const WIN32_MIN_ABS_FILEURL_LENGTH: usize = "file:///C:/".len(); pub fn extract_drive_letter(url_base: &str, path: &str) -> super::Result<(String, String)> { if path.len() < 4 || !":|".contains(path.chars().nth(2).unwrap()) || path.chars().nth(3).unwrap() != '/' { return Err(super::Error::InvalidWin32Path(path.to_owned())); } let url_base = url_base.to_owned() + &path[0..3]; let path = &path[3..]; Ok((url_base, path.to_owned())) } pub fn strip_local_trailing_slash(url: &str) -> String { if url.len() > WIN32_MIN_ABS_FILEURL_LENGTH { url[..url.len() - 1].to_owned() } else { url.to_owned() } } } pub mod posix { use std::path::{Path, PathBuf}; /// Convert a local path like ./foo into a URL like file:///path/to/foo /// /// This also handles transforming escaping unicode characters, etc. pub fn local_path_to_url>(path: P) -> std::io::Result { let abs_path = crate::osutils::path::posix::abspath(path.as_ref())?; let escaped_path = super::escape( abs_path.as_path().as_os_str().as_encoded_bytes(), Some("/~"), ); Ok(format!("file://{}", escaped_path)) } const FILE_LOCALHOST_PREFIX: &str = "file://localhost"; const PLAIN_FILE_PREFIX: &str = "file:///"; pub fn local_path_from_url(url: &str) -> std::result::Result { let url = super::strip_segment_parameters(url); let path = if let Some(suffix) = url.strip_prefix(FILE_LOCALHOST_PREFIX) { suffix } else if url.starts_with(PLAIN_FILE_PREFIX) { &url[PLAIN_FILE_PREFIX.len() - 1..] } else { return Err(super::Error::NotLocalUrl(url.to_string())); }; Ok(PathBuf::from(super::unescape(path)?)) } } pub fn local_path_to_url>(path: P) -> std::io::Result { #[cfg(target_os = "windows")] return Ok(win32::local_path_to_url(path)?); #[cfg(unix)] return posix::local_path_to_url(path); } pub fn local_path_from_url(url: &str) -> Result { #[cfg(target_os = "windows")] return Ok(win32::local_path_from_url(url)?); #[cfg(unix)] return posix::local_path_from_url(url); } /// Derive a TO_LOCATION given a FROM_LOCATION. /// /// The normal case is a FROM_LOCATION of http://foo/bar => bar. /// The Right Thing for some logical destinations may differ though /// because no / may be present at all. In that case, the result is /// the full name without the scheme indicator, e.g. lp:foo-bar => foo-bar. /// This latter case also applies when a Windows drive /// is used without a path, e.g. c:foo-bar => foo-bar. /// If no /, path separator or : is found, the from_location is returned. pub fn derive_to_location(from_location: &str) -> String { let from_location = strip_segment_parameters(from_location); if let Some(separator_index) = from_location.rfind('/') { let basename = &from_location[separator_index + 1..]; basename.trim_end_matches("/\\").to_string() } else if let Some(separator_index) = from_location.find(':') { return from_location[separator_index + 1..].to_string(); } else { return from_location.to_string(); } } #[cfg(target_os = "windows")] pub const MIN_ABS_FILEURL_LENGTH: usize = "file:///C:".len(); #[cfg(not(target_os = "windows"))] pub const MIN_ABS_FILEURL_LENGTH: usize = "file:///".len(); /// Compute just the relative sub-portion of a url /// /// This assumes that both paths are already fully specified file:// URLs. pub fn file_relpath(base: &str, path: &str) -> Result { if base.len() < MIN_ABS_FILEURL_LENGTH { return Err(Error::UrlTooShort(base.to_string())); } let base: PathBuf = crate::osutils::path::normpath(local_path_from_url(base)?); let path: PathBuf = crate::osutils::path::normpath(local_path_from_url(path)?); let relpath = crate::osutils::path::relpath(base.as_path(), path.as_path()); if relpath.is_none() { return Err(Error::PathNotChild( path.display().to_string(), base.display().to_string(), )); } Ok(escape( relpath.unwrap().as_os_str().as_encoded_bytes(), None, )) } /// Run the URL_HEX_ESCAPES_RE pass over a path, decoding "safe" hex /// escapes (alphanumerics + `-._~`) and uppercasing the rest. Mirrors /// the `_url_hex_escapes_re.sub(_unescape_safe_chars, ...)` used by the /// Python URL.__init__. pub fn normalize_quoted_path(path: &str) -> String { URL_HEX_ESCAPES_RE .replace_all(path, unescape_safe_chars) .to_string() } /// Decoded URL components, mirroring dromedary.urlutils.URL. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ParsedUrl { pub scheme: String, pub quoted_user: Option, pub quoted_password: Option, pub quoted_host: String, pub port: Option, pub quoted_path: String, } /// Split a URL into (scheme, netloc, path), matching urlparse.urlparse /// with `allow_fragments=False`. The path includes the leading `/`. fn split_scheme_netloc_path(url: &str) -> (String, String, String) { // Find scheme: characters up to the first ':' followed by '//'. let scheme_end = url.find(':'); let (scheme, rest) = match scheme_end { Some(i) if url[i + 1..].starts_with("//") => (url[..i].to_string(), &url[i + 3..]), _ => return (String::new(), String::new(), url.to_string()), }; // After ://, netloc runs up to the next '/' (or end). match rest.find('/') { Some(j) => (scheme, rest[..j].to_string(), rest[j..].to_string()), None => (scheme, rest.to_string(), String::new()), } } /// Parse a URL into its quoted components. Mirrors /// dromedary.urlutils.URL.from_string and parse_url. Quoted forms are /// preserved verbatim — no percent decoding is done here. pub fn parse_url(url: &str) -> Result { let (scheme, netloc, path) = split_scheme_netloc_path(url); // Pull out user[:password]@ if present. let (user, password, host_part) = if let Some(at) = netloc.rfind('@') { let user_part = &netloc[..at]; let host_part = &netloc[at + 1..]; let (u, p) = match user_part.find(':') { Some(c) => (&user_part[..c], Some(&user_part[c + 1..])), None => (user_part, None), }; (Some(u.to_string()), p.map(|s| s.to_string()), host_part) } else { (None, None, netloc.as_str()) }; // Extract port if there's a `:` in the host portion AND the host // isn't a bracketed IPv6 literal (which itself contains colons). let mut host = host_part.to_string(); let mut port: Option = None; let bracketed = host.starts_with('[') && host.ends_with(']'); if host.contains(':') && !bracketed { if let Some(c) = host.rfind(':') { let port_str = host[c + 1..].to_string(); host.truncate(c); if !port_str.is_empty() { port = Some( port_str .parse::() .map_err(|_| Error::InvalidUrlPort(url.to_string(), port_str.clone()))?, ); } } } // Strip the brackets off an IPv6 literal once port handling is done. if host.starts_with('[') && host.ends_with(']') && host.len() >= 2 { host = host[1..host.len() - 1].to_string(); } Ok(ParsedUrl { scheme, quoted_user: user, quoted_password: password, quoted_host: host, port, quoted_path: path, }) } #[cfg(test)] mod parse_url_tests { use super::*; #[test] fn simple() { let p = parse_url("http://example.com:80/one").unwrap(); assert_eq!(p.scheme, "http"); assert_eq!(p.quoted_user, None); assert_eq!(p.quoted_password, None); assert_eq!(p.quoted_host, "example.com"); assert_eq!(p.port, Some(80)); assert_eq!(p.quoted_path, "/one"); } #[test] fn ipv6() { let p = parse_url("http://[1:2:3::40]/one").unwrap(); assert_eq!(p.quoted_host, "1:2:3::40"); assert_eq!(p.port, None); assert_eq!(p.quoted_path, "/one"); } #[test] fn ipv6_with_port() { let p = parse_url("http://[1:2:3::40]:80/one").unwrap(); assert_eq!(p.quoted_host, "1:2:3::40"); assert_eq!(p.port, Some(80)); } #[test] fn user_password() { let p = parse_url("http://ro%62ey:h%40t@ex%41mple.com:2222/path").unwrap(); assert_eq!(p.quoted_user.as_deref(), Some("ro%62ey")); assert_eq!(p.quoted_password.as_deref(), Some("h%40t")); assert_eq!(p.quoted_host, "ex%41mple.com"); assert_eq!(p.port, Some(2222)); assert_eq!(p.quoted_path, "/path"); } #[test] fn empty_port() { let p = parse_url("http://example.com:/one").unwrap(); assert_eq!(p.quoted_host, "example.com"); assert_eq!(p.port, None); } #[test] fn invalid_port() { match parse_url("http://example.com:abc/one") { Err(Error::InvalidUrlPort(_, port_str)) => assert_eq!(port_str, "abc"), other => panic!("expected InvalidUrlPort, got {:?}", other), } } #[test] fn normalize_quoted_path_unescapes_safe_chars() { // %7E is ~, which is unreserved → unescape to literal. assert_eq!(normalize_quoted_path("/foo%7Ebar"), "/foo~bar"); // %40 is @, which is reserved → keep as %40 (uppercased). assert_eq!(normalize_quoted_path("/foo%40bar"), "/foo%40bar"); // Lowercase hex → uppercase. assert_eq!(normalize_quoted_path("/foo%2fbar"), "/foo%2Fbar"); } } dromedary-0.1.1/src/win32-locks.rs000064400000000000000000000170641046102023000150310ustar 00000000000000//! Windows file locking. //! //! This uses `std::fs::File::try_lock` / `File::lock` (stable in Rust 1.89) //! to take advisory OS-level locks via `LockFileEx`, combined with the same //! process-local tracking tables that `fcntl-locks.rs` uses so that a single //! process can still distinguish read vs. write contention. //! //! TODO(windows): `File::try_lock` is exclusive-only. Implementing shared //! read locks that interoperate with other processes requires dropping to //! `LockFileEx` directly via `windows-sys`. For pass 1 we rely on the //! process-local tracking alone for read-vs-write arbitration, which is //! sufficient for single-process tests but will not coordinate across //! processes for shared reads. use crate::lock::{FileLock, Lock, LockError}; use lazy_static::lazy_static; use log::debug; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::fs::{File, OpenOptions}; use std::path::{Path, PathBuf}; fn open(filename: &Path, options: &OpenOptions) -> std::result::Result<(PathBuf, File), LockError> { let filename = crate::osutils::path::realpath(filename)?; match options.open(&filename) { Ok(f) => Ok((filename, f)), Err(e) => match e.kind() { std::io::ErrorKind::PermissionDenied => Err(LockError::Failed(filename, e.to_string())), std::io::ErrorKind::NotFound => { debug!( "trying to create missing lock {}", filename.to_string_lossy() ); let f = OpenOptions::new() .create(true) .write(true) .read(true) .open(&filename)?; Ok((filename, f)) } _ => Err(e.into()), }, } } lazy_static! { static ref OPEN_WRITE_LOCKS: std::sync::Mutex> = std::sync::Mutex::new(HashSet::new()); static ref OPEN_READ_LOCKS: std::sync::Mutex> = std::sync::Mutex::new(HashMap::new()); } pub struct WriteLock { filename: PathBuf, f: File, } impl WriteLock { pub fn new(filename: &Path, strict_locks: bool) -> Result { let filename = crate::osutils::path::realpath(filename)?; if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { return Err(LockError::Contention(filename)); } if OPEN_READ_LOCKS.lock().unwrap().contains_key(&filename) { if strict_locks { return Err(LockError::Contention(filename)); } else { debug!( "Write lock taken w/ an open read lock on: {}", filename.to_string_lossy() ); } } let (filename, f) = open( filename.as_path(), OpenOptions::new().read(true).write(true), )?; match f.try_lock() { Ok(()) => {} Err(std::fs::TryLockError::WouldBlock) => { return Err(LockError::Contention(filename)); } Err(std::fs::TryLockError::Error(_)) => { // Fall through — we still have process-local tracking. } } OPEN_WRITE_LOCKS.lock().unwrap().insert(filename.clone()); Ok(WriteLock { filename, f }) } } impl Lock for WriteLock { fn unlock(&mut self) -> Result<(), LockError> { OPEN_WRITE_LOCKS.lock().unwrap().remove(&self.filename); let _ = self.f.unlock(); Ok(()) } } impl FileLock for WriteLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } pub struct ReadLock { filename: PathBuf, f: File, } impl ReadLock { pub fn new(filename: &Path, strict_locks: bool) -> std::result::Result { let filename = crate::osutils::path::realpath(filename)?; if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { if strict_locks { return Err(LockError::Contention(filename)); } else { debug!( "Read lock taken w/ an open write lock on: {}", filename.to_string_lossy() ); } } OPEN_READ_LOCKS .lock() .unwrap() .entry(filename.clone()) .and_modify(|count| *count += 1) .or_insert(1); let (filename, f) = open(&filename, OpenOptions::new().read(true))?; // `File::try_lock_shared` would be the right call here, but it is // currently unstable. See the module-level TODO. Ok(ReadLock { filename, f }) } /// Try to grab a write lock on the file. pub fn temporary_write_lock( self, ) -> std::result::Result { if OPEN_WRITE_LOCKS.lock().unwrap().contains(&self.filename) { panic!("file already locked: {}", self.filename.to_string_lossy()); } TemporaryWriteLock::new(self) } } impl Lock for ReadLock { fn unlock(&mut self) -> std::result::Result<(), LockError> { match OPEN_READ_LOCKS.lock().unwrap().entry(self.filename.clone()) { Entry::Occupied(mut entry) => { let count = entry.get_mut(); if *count == 1 { entry.remove(); } else { *count -= 1; } } Entry::Vacant(_) => panic!("no read lock on {}", self.filename.to_string_lossy()), } Ok(()) } } impl FileLock for ReadLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } /// A token used when grabbing a temporary_write_lock. pub struct TemporaryWriteLock { read_lock: ReadLock, filename: PathBuf, f: File, } impl TemporaryWriteLock { pub fn new(read_lock: ReadLock) -> std::result::Result { let filename = read_lock.filename.clone(); if let Some(count) = OPEN_READ_LOCKS.lock().unwrap().get(&filename) { if *count > 1 { return Err((read_lock, LockError::Contention(filename))); } } if OPEN_WRITE_LOCKS.lock().unwrap().contains(&filename) { panic!("file already locked: {}", filename.to_string_lossy()); } let f = match OpenOptions::new() .write(true) .read(true) .create(true) .open(&filename) { Ok(f) => Ok(f), Err(e) => return Err((read_lock, e.into())), }?; match f.try_lock() { Ok(()) => {} Err(std::fs::TryLockError::WouldBlock) => { return Err((read_lock, LockError::Contention(filename))); } Err(std::fs::TryLockError::Error(_)) => { // Fall through — process-local tracking is what we rely on. } } OPEN_WRITE_LOCKS.lock().unwrap().insert(filename.clone()); Ok(Self { read_lock, filename, f, }) } pub fn restore_read_lock(self) -> ReadLock { let _ = self.f.unlock(); OPEN_WRITE_LOCKS.lock().unwrap().remove(&self.filename); self.read_lock } } impl FileLock for TemporaryWriteLock { fn file(&self) -> std::io::Result> { Ok(Box::new(self.f.try_clone()?)) } fn path(&self) -> &Path { &self.filename } } dromedary-0.1.1/ssh/__init__.py000064400000000000000000001036601046102023000145250ustar 00000000000000# Copyright (C) 2006-2011 Robey Pointer # Copyright (C) 2005, 2006, 2007 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Foundation SSH support for SFTP and smart server.""" import errno import logging import os import socket import subprocess import sys from binascii import hexlify from catalogus import registry from dromedary import _bedding as bedding from dromedary import _config, _ui, errors from dromedary.errors import SocketConnectionError, StrangeHostname from dromedary.osutils import get_terminal_encoding, pathjoin, set_fd_cloexec from .._transport_rs import sftp as _sftp_rs logger = logging.getLogger("dromedary.ssh") SFTPClient = _sftp_rs.SFTPClient try: import paramiko except ModuleNotFoundError: # If we have an ssh subprocess, we don't strictly need paramiko for all ssh # access paramiko = None # type: ignore class SSHVendorNotFound(errors.TransportError): """No SSH implementation available.""" _fmt = ( "Don't know how to handle SSH connections." " Please set BRZ_SSH environment variable." ) class UnknownSSH(errors.TransportError): """Unknown SSH implementation specified.""" _fmt = "Unrecognised value for BRZ_SSH environment variable: %(vendor)s" def __init__(self, vendor): """Initialize with the unrecognised vendor name.""" self.vendor = vendor errors.TransportError.__init__(self) class SSHVendorManager(registry.Registry[str, "SSHVendor", None]): """Manager for manage SSH vendors.""" def __init__(self): """Initialize the SSH vendor manager. Sets up the registry and initializes the vendor cache. """ super().__init__() self._cached_ssh_vendor = None def clear_cache(self): """Clear previously cached lookup result.""" self._cached_ssh_vendor = None def _get_vendor_by_config(self): """Get SSH vendor based on configuration. Looks up the SSH vendor from the global configuration. If a vendor name is specified but not registered, attempts to use it as an executable path. Returns: SSHVendor: The configured SSH vendor, or None if not configured. Raises: UnknownSSH: If the configured vendor name is not found and cannot be used as an executable path. """ vendor_name = _config.get_ssh_vendor_name() if vendor_name is not None: try: vendor = self.get(vendor_name) except KeyError as err: vendor = self._get_vendor_from_path(vendor_name) if vendor is None: raise UnknownSSH(vendor_name) from err vendor.executable_path = vendor_name return vendor return None def _get_ssh_version_string(self, args): """Return SSH version string from the subprocess. Runs the given command and captures its output to determine the SSH implementation version. Args: args: Command line arguments to execute (typically ['ssh', '-V']). Returns: str: Combined stdout and stderr output decoded using terminal encoding. Returns empty string if the command fails. """ try: p = subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, **os_specific_subprocess_params(), ) stdout, stderr = p.communicate() except OSError: stdout = stderr = b"" return (stdout + stderr).decode(get_terminal_encoding()) def _get_vendor_by_version_string(self, version, progname): """Return the vendor or None based on output from the subprocess. Examines the version string to determine which SSH implementation is being used (OpenSSH, SSH Corp, GNU lsh, or PuTTY plink). Args: version: The output of 'ssh -V' like command. progname: The program name that was executed. Returns: SSHVendor: The appropriate vendor instance, or None if not recognized. """ vendor = None if "OpenSSH" in version: logger.debug("ssh implementation is OpenSSH") vendor = OpenSSHSubprocessVendor() elif "SSH Secure Shell" in version: logger.debug("ssh implementation is SSH Corp.") vendor = SSHCorpSubprocessVendor() elif "lsh" in version: logger.debug("ssh implementation is GNU lsh.") vendor = LSHSubprocessVendor() # As plink user prompts are not handled currently, don't auto-detect # it by inspection below, but keep this vendor detection for if a path # is given in BRZ_SSH. See https://bugs.launchpad.net/bugs/414743 elif "plink" in version and progname == "plink": # Checking if "plink" was the executed argument as Windows # sometimes reports 'ssh -V' incorrectly with 'plink' in its # version. See https://bugs.launchpad.net/bzr/+bug/107155 logger.debug("ssh implementation is Putty's plink.") vendor = PLinkSubprocessVendor() return vendor def _get_vendor_by_inspection(self): """Return the vendor or None by checking for known SSH implementations. Runs 'ssh -V' to determine the SSH implementation in use. Returns: SSHVendor: The detected vendor, or None if not recognized. """ version = self._get_ssh_version_string(["ssh", "-V"]) return self._get_vendor_by_version_string(version, "ssh") def _get_vendor_from_path(self, path): """Return the vendor or None using the program at the given path. Runs the specified executable with '-V' to determine its type. Args: path: Path to the SSH executable. Returns: SSHVendor: The detected vendor, or None if not recognized. """ version = self._get_ssh_version_string([path, "-V"]) return self._get_vendor_by_version_string( version, os.path.splitext(os.path.basename(path))[0] ) def get_vendor(self): """Find out what version of SSH is on the system. :raises SSHVendorNotFound: if no any SSH vendor is found :raises UnknownSSH: if the BRZ_SSH environment variable contains unknown vendor name """ if self._cached_ssh_vendor is None: vendor = self._get_vendor_by_config() if vendor is None: vendor = self._get_vendor_by_inspection() if vendor is None: logger.debug("falling back to default implementation") if self.default_key is None: raise SSHVendorNotFound() vendor = self.get() self._cached_ssh_vendor = vendor return self._cached_ssh_vendor _ssh_vendor_manager = SSHVendorManager() _get_ssh_vendor = _ssh_vendor_manager.get_vendor register_ssh_vendor = _ssh_vendor_manager.register register_lazy_ssh_vendor = _ssh_vendor_manager.register_lazy def _ignore_signals(): """Configure signal handling for SSH subprocesses. Sets up signal handlers to ignore SIGINT and conditionally SIGQUIT. This prevents the SSH subprocess from being interrupted by keyboard interrupts intended for the parent process. The function ignores: - SIGINT: Always ignored to prevent Ctrl+C from affecting SSH - SIGQUIT: Ignored if not already set to default (to respect breakin) """ # TODO: This should possibly ignore SIGHUP as well, but bzr currently # doesn't handle it itself. # import signal signal.signal(signal.SIGINT, signal.SIG_IGN) # GZ 2010-02-19: Perhaps make this check if breakin is installed instead if signal.getsignal(signal.SIGQUIT) != signal.SIG_DFL: signal.signal(signal.SIGQUIT, signal.SIG_IGN) class SocketAsChannelAdapter: """Simple wrapper for a socket that pretends to be a paramiko Channel.""" def __init__(self, sock): """Initialize the adapter with a socket. Args: sock: A socket object to wrap. """ self.__socket = sock def get_name(self): """Get the name of this channel adapter. Returns: str: A descriptive name for this adapter. """ return "bzr SocketAsChannelAdapter" def send(self, data): """Send data through the socket. Args: data: Bytes to send. Returns: int: Number of bytes sent. """ return self.__socket.send(data) def recv(self, n): """Receive data from the socket. Args: n: Maximum number of bytes to receive. Returns: bytes: Data received from the socket, or empty string if the connection is closed. Note: Returns empty string instead of raising an exception when the connection is closed, to match paramiko's expected behavior. """ try: return self.__socket.recv(n) except OSError as e: if e.args[0] in ( errno.EPIPE, errno.ECONNRESET, errno.ECONNABORTED, errno.EBADF, ): # Connection has closed. Paramiko expects an empty string in # this case, not an exception. return "" raise def recv_ready(self): """Check if data is available for reading. Returns: bool: Always returns True. Should ideally use poll() or select() to check for actual data availability. Note: This is a simplified implementation that always returns True. A proper implementation would check if data is actually available. """ # TODO: jam 20051215 this function is necessary to support the # pipelined() function. In reality, it probably should use # poll() or select() to actually return if there is data # available, otherwise we probably don't get any benefit return True def close(self): """Close the underlying socket.""" self.__socket.close() class SSHVendor: """Abstract base class for SSH vendor implementations.""" def connect_sftp(self, username, password, host, port): """Make an SSH connection, and return an SFTPClient. :param username: an ascii string :param password: an ascii string :param host: a host name as an ascii string :param port: a port number :type port: int :raises: ConnectionError if it cannot connect. :rtype: paramiko.sftp_client.SFTPClient """ raise NotImplementedError(self.connect_sftp) def connect_ssh(self, username, password, host, port, command): """Make an SSH connection. :returns: an SSHConnection. """ raise NotImplementedError(self.connect_ssh) def _raise_connection_error( self, host, port=None, orig_error=None, msg="Unable to connect to SSH host" ): """Raise a SocketConnectionError with properly formatted host. This just unifies all the locations that try to raise ConnectionError, so that they format things properly. Args: host: The hostname that failed to connect. port: The port number (optional). orig_error: The original exception that caused the connection failure. msg: Custom error message. Raises: SocketConnectionError: Always raises this error with the provided details. """ raise SocketConnectionError( host=host, port=port, msg=msg, orig_error=orig_error ) class LoopbackVendor(SSHVendor): """SSH "vendor" that connects over a plain TCP socket, not SSH.""" def connect_sftp(self, username, password, host, port): """Connect to an SFTP server using a plain TCP socket. This is a loopback implementation that bypasses SSH and connects directly via TCP. Useful for testing or local connections. Args: username: SSH username (ignored in loopback). password: SSH password (ignored in loopback). host: Hostname to connect to. port: Port number to connect to. Returns: SFTPClient: An SFTP client connected via TCP socket. Raises: SocketConnectionError: If connection fails. """ sock = socket.socket() try: sock.connect((host, port)) except OSError as e: self._raise_connection_error(host, port=port, orig_error=e) return SFTPClient(sock.detach()) register_ssh_vendor("loopback", LoopbackVendor()) _ssh_connection_errors: tuple[type[Exception], ...] = ( EOFError, OSError, IOError, socket.error, ) if paramiko is not None: register_lazy_ssh_vendor("paramiko", "dromedary.ssh.paramiko", "paramiko_vendor") register_lazy_ssh_vendor("none", "dromedary.ssh.paramiko", "paramiko_vendor") _ssh_vendor_manager.default_key = "paramiko" _ssh_connection_errors += (paramiko.SSHException,) class SubprocessVendor(SSHVendor): """Abstract base class for vendors that use pipes to a subprocess.""" # In general stderr should be inherited from the parent process so prompts # are visible on the terminal. This can be overriden to another file for # tests, but beware of using PIPE which may hang due to not being read. _stderr_target = None @staticmethod def _check_hostname(arg): """Check if hostname is safe to pass to subprocess. Args: arg: The hostname to check. Raises: StrangeHostname: If the hostname starts with a dash. """ if arg.startswith("-"): raise StrangeHostname(hostname=arg) def _connect(self, argv): """Create and connect to an SSH subprocess. Attempts to use socketpair for better performance (non-blocking reads), falling back to pipes if socketpair is not available. Args: argv: Command line arguments for the SSH subprocess. Returns: SSHSubprocessConnection: A connection to the SSH subprocess. """ # Attempt to make a socketpair to use as stdin/stdout for the SSH # subprocess. We prefer sockets to pipes because they support # non-blocking short reads, allowing us to optimistically read 64k (or # whatever) chunks. try: my_sock, subproc_sock = socket.socketpair() set_fd_cloexec(my_sock) except (AttributeError, OSError): # This platform doesn't support socketpair(), so just use ordinary # pipes instead. stdin = stdout = subprocess.PIPE my_sock, subproc_sock = None, None else: stdin = stdout = subproc_sock proc = subprocess.Popen( argv, stdin=stdin, stdout=stdout, stderr=self._stderr_target, bufsize=0, **os_specific_subprocess_params(), ) if subproc_sock is not None: subproc_sock.close() return SSHSubprocessConnection(proc, sock=my_sock) def connect_sftp(self, username, password, host, port): """Connect to an SFTP server using an SSH subprocess. Args: username: SSH username. password: SSH password (not used by subprocess vendors). host: Hostname to connect to. port: Port number to connect to. Returns: SFTPClient: An SFTP client connected via SSH subprocess. Raises: SocketConnectionError: If connection fails. """ try: argv = self._get_vendor_specific_argv( username, host, port, subsystem="sftp" ) sock = self._connect(argv) return SFTPClient(sock._sock.detach()) except _ssh_connection_errors as e: self._raise_connection_error(host, port=port, orig_error=e) def connect_ssh(self, username, password, host, port, command): """Connect to an SSH server and run a command. Args: username: SSH username. password: SSH password (not used by subprocess vendors). host: Hostname to connect to. port: Port number to connect to. command: Command to execute on the remote host. Returns: SSHSubprocessConnection: A connection to the SSH subprocess. Raises: SocketConnectionError: If connection fails. """ try: argv = self._get_vendor_specific_argv(username, host, port, command=command) return self._connect(argv) except _ssh_connection_errors as e: self._raise_connection_error(host, port=port, orig_error=e) def _get_vendor_specific_argv( self, username, host, port, subsystem=None, command=None ): """Returns the argument list to run the subprocess with. Exactly one of 'subsystem' and 'command' must be specified. """ raise NotImplementedError(self._get_vendor_specific_argv) class OpenSSHSubprocessVendor(SubprocessVendor): """SSH vendor that uses the 'ssh' executable from OpenSSH.""" executable_path = "ssh" def _get_vendor_specific_argv( self, username, host, port, subsystem=None, command=None ): """Build OpenSSH command line arguments. Args: username: SSH username. host: Hostname to connect to. port: Port number (optional). subsystem: SSH subsystem to invoke (e.g., 'sftp'). command: Command to execute (alternative to subsystem). Returns: list: Command line arguments for OpenSSH. """ args = [ self.executable_path, "-oForwardX11=no", "-oForwardAgent=no", "-oClearAllForwardings=yes", "-oNoHostAuthenticationForLocalhost=yes", ] if port is not None: args.extend(["-p", str(port)]) if username is not None: args.extend(["-l", username]) if subsystem is not None: args.extend(["-s", "--", host, subsystem]) else: args.extend(["--", host] + command) return args register_ssh_vendor("openssh", OpenSSHSubprocessVendor()) class SSHCorpSubprocessVendor(SubprocessVendor): """SSH vendor that uses the 'ssh' executable from SSH Corporation.""" executable_path = "ssh" def _get_vendor_specific_argv( self, username, host, port, subsystem=None, command=None ): """Build SSH Corporation command line arguments. Args: username: SSH username. host: Hostname to connect to. port: Port number (optional). subsystem: SSH subsystem to invoke (e.g., 'sftp'). command: Command to execute (alternative to subsystem). Returns: list: Command line arguments for SSH Corp's ssh. """ self._check_hostname(host) args = [self.executable_path, "-x"] if port is not None: args.extend(["-p", str(port)]) if username is not None: args.extend(["-l", username]) if subsystem is not None: args.extend(["-s", subsystem, host]) else: args.extend([host] + command) return args register_ssh_vendor("sshcorp", SSHCorpSubprocessVendor()) class LSHSubprocessVendor(SubprocessVendor): """SSH vendor that uses the 'lsh' executable from GNU.""" executable_path = "lsh" def _get_vendor_specific_argv( self, username, host, port, subsystem=None, command=None ): """Build GNU lsh command line arguments. Args: username: SSH username. host: Hostname to connect to. port: Port number (optional). subsystem: SSH subsystem to invoke (e.g., 'sftp'). command: Command to execute (alternative to subsystem). Returns: list: Command line arguments for GNU lsh. """ self._check_hostname(host) args = [self.executable_path] if port is not None: args.extend(["-p", str(port)]) if username is not None: args.extend(["-l", username]) if subsystem is not None: args.extend(["--subsystem", subsystem, host]) else: args.extend([host] + command) return args register_ssh_vendor("lsh", LSHSubprocessVendor()) class PLinkSubprocessVendor(SubprocessVendor): """SSH vendor that uses the 'plink' executable from Putty.""" executable_path = "plink" def _get_vendor_specific_argv( self, username, host, port, subsystem=None, command=None ): """Build PuTTY plink command line arguments. Args: username: SSH username. host: Hostname to connect to. port: Port number (optional). subsystem: SSH subsystem to invoke (e.g., 'sftp'). command: Command to execute (alternative to subsystem). Returns: list: Command line arguments for plink. """ self._check_hostname(host) args = [self.executable_path, "-x", "-a", "-ssh", "-2", "-batch"] if port is not None: args.extend(["-P", str(port)]) if username is not None: args.extend(["-l", username]) if subsystem is not None: args.extend(["-s", host, subsystem]) else: args.extend([host] + command) return args register_ssh_vendor("plink", PLinkSubprocessVendor()) def _paramiko_auth(username, password, host, port, paramiko_transport): # paramiko requires a username, but it might be none if nothing was # supplied. If so, use the local username. if username is None: username = _config.get_auth_user("ssh", host, port=port) agent = paramiko.Agent() for key in agent.get_keys(): logger.debug("Trying SSH agent key %s", hexlify(key.get_fingerprint()).upper()) try: paramiko_transport.auth_publickey(username, key) return except paramiko.SSHException: pass # okay, try finding id_rsa or id_dss? (posix only) if _try_pkey_auth(paramiko_transport, paramiko.RSAKey, username, "id_rsa"): return # DSSKey was removed in paramiko 4.0.0 as DSA keys are deprecated if hasattr(paramiko, "DSSKey"): if _try_pkey_auth(paramiko_transport, paramiko.DSSKey, username, "id_dsa"): return # If we have gotten this far, we are about to try for passwords, do an # auth_none check to see if it is even supported. supported_auth_types = [] try: # Note that with paramiko <1.7.5 this logs an INFO message: # Authentication type (none) not permitted. # So we explicitly disable the logging level for this action old_level = paramiko_transport.logger.level paramiko_transport.logger.setLevel(logging.WARNING) try: paramiko_transport.auth_none(username) finally: paramiko_transport.logger.setLevel(old_level) except paramiko.BadAuthenticationType as e: # Supported methods are in the exception supported_auth_types = e.allowed_types except paramiko.SSHException: # Don't know what happened, but just ignore it pass # We treat 'keyboard-interactive' and 'password' auth methods identically, # because Paramiko's auth_password method will automatically try # 'keyboard-interactive' auth (using the password as the response) if # 'password' auth is not available. Apparently some Debian and Gentoo # OpenSSH servers require this. # XXX: It's possible for a server to require keyboard-interactive auth that # requires something other than a single password, but we currently don't # support that. if ( "password" not in supported_auth_types and "keyboard-interactive" not in supported_auth_types ): raise errors.ConnectionError( "Unable to authenticate to SSH host as" "\n {}@{}\nsupported auth types: {}".format( username, host, supported_auth_types ) ) if password: try: paramiko_transport.auth_password(username, password) return except paramiko.SSHException: pass # give up and ask for a password password = _config.get_auth_password("ssh", host, username, port=port) # get_password can still return None, which means we should not prompt if password is not None: try: paramiko_transport.auth_password(username, password) except paramiko.SSHException as e: raise errors.ConnectionError( "Unable to authenticate to SSH host as\n {}@{}\n".format( username, host ), e, ) from e else: raise errors.ConnectionError( "Unable to authenticate to SSH host as {}@{}".format(username, host) ) def _try_pkey_auth(paramiko_transport, pkey_class, username, filename): filename = os.path.expanduser("~/.ssh/" + filename) try: key = pkey_class.from_private_key_file(filename) paramiko_transport.auth_publickey(username, key) return True except paramiko.PasswordRequiredException: password = _ui.get_password( "SSH %(filename)s password", filename=os.fsdecode(filename) ) try: key = pkey_class.from_private_key_file(filename, password) paramiko_transport.auth_publickey(username, key) return True except paramiko.SSHException: logger.debug( "SSH authentication via %s key failed.", os.path.basename(filename), ) except paramiko.SSHException: logger.debug( "SSH authentication via %s key failed.", os.path.basename(filename) ) except OSError: pass return False def _ssh_host_keys_config_dir(): return pathjoin(bedding.config_dir(), "ssh_host_keys") def load_host_keys(): """Load system host keys (probably doesn't work on windows) and any "discovered" keys from previous sessions. """ global SYSTEM_HOSTKEYS, BRZ_HOSTKEYS try: SYSTEM_HOSTKEYS = paramiko.util.load_host_keys( os.path.expanduser("~/.ssh/known_hosts") ) except OSError as e: logger.debug("failed to load system host keys: %s", e) brz_hostkey_path = _ssh_host_keys_config_dir() try: BRZ_HOSTKEYS = paramiko.util.load_host_keys(brz_hostkey_path) except OSError as e: logger.debug("failed to load brz host keys: %s", e) save_host_keys() def save_host_keys(): """Save "discovered" host keys in $(config)/ssh_host_keys/.""" global SYSTEM_HOSTKEYS, BRZ_HOSTKEYS bzr_hostkey_path = _ssh_host_keys_config_dir() bedding.ensure_config_dir_exists() try: with open(bzr_hostkey_path, "w") as f: f.write("# SSH host keys collected by bzr\n") for hostname, keys in BRZ_HOSTKEYS.items(): for keytype, key in keys.items(): f.write("{} {} {}\n".format(hostname, keytype, key.get_base64())) except OSError as e: logger.debug("failed to save bzr host keys: %s", e) def os_specific_subprocess_params(): """Get O/S specific subprocess parameters. Returns different parameters based on the operating system: - Windows: Empty dict (no special handling) - Unix-like: Dict with preexec_fn to ignore signals and close_fds=True Returns: dict: Subprocess parameters suitable for the current OS. """ if sys.platform == "win32": # setting the process group and closing fds is not supported on # win32 return {} else: # We close fds other than the pipes as the child process does not need # them to be open. # # We also set the child process to ignore SIGINT. Normally the signal # would be sent to every process in the foreground process group, but # this causes it to be seen only by bzr and not by ssh. Python will # generate a KeyboardInterrupt in bzr, and we will then have a chance # to release locks or do other cleanup over ssh before the connection # goes away. # # # Running it in a separate process group is not good because then it # can't get non-echoed input of a password or passphrase. # return { "preexec_fn": _ignore_signals, "close_fds": True, } import weakref _subproc_weakrefs: set[weakref.ref] = set() def _close_ssh_proc(proc, sock): """Carefully close stdin/stdout and reap the SSH process. If the pipes are already closed and/or the process has already been wait()ed on, that's ok, and no error is raised. The goal is to do our best to clean up (whether or not a clean up was already tried). Args: proc: The subprocess.Popen instance to clean up. sock: Optional socket to close (may be None). Note: Silently ignores OSError exceptions during cleanup to handle already-closed resources gracefully. """ funcs = [] for closeable in (proc.stdin, proc.stdout, sock): # We expect that either proc (a subprocess.Popen) will have stdin and # stdout streams to close, or that we will have been passed a socket to # close, with the option not in use being None. if closeable is not None: funcs.append(closeable.close) funcs.append(proc.wait) for func in funcs: try: func() except OSError: # It's ok for the pipe to already be closed, or the process to # already be finished. continue class SSHConnection: """Abstract base class for SSH connections.""" def get_sock_or_pipes(self): """Returns a (kind, io_object) pair. If kind == 'socket', then io_object is a socket. If kind == 'pipes', then io_object is a pair of file-like objects (read_from, write_to). Returns: tuple: A (kind, io_object) pair where: - kind is either 'socket' or 'pipes' - io_object is either a socket or (read_file, write_file) tuple """ raise NotImplementedError(self.get_sock_or_pipes) def close(self): """Close the SSH connection. Subclasses must implement this method to properly close their connection type. """ raise NotImplementedError(self.close) class SSHSubprocessConnection(SSHConnection): """A connection to an ssh subprocess via pipes or a socket. This class is also socket-like enough to be used with SocketAsChannelAdapter (it has 'send' and 'recv' methods). """ def __init__(self, proc, sock=None): """Constructor. :param proc: a subprocess.Popen :param sock: if proc.stdin/out is a socket from a socketpair, then sock should breezy's half of that socketpair. If not passed, proc's stdin/out is assumed to be ordinary pipes. """ self.proc = proc self._sock = sock # Add a weakref to proc that will attempt to do the same as self.close # to avoid leaving processes lingering indefinitely. def terminate(ref): _subproc_weakrefs.remove(ref) _close_ssh_proc(proc, sock) _subproc_weakrefs.add(weakref.ref(self, terminate)) def send(self, data): """Send data through the connection. Uses either socket.send() or os.write() depending on the connection type. Args: data: Bytes to send. Returns: int: Number of bytes sent. """ if self._sock is not None: return self._sock.send(data) else: return os.write(self.proc.stdin.fileno(), data) def recv(self, count): """Receive data from the connection. Uses either socket.recv() or os.read() depending on the connection type. Args: count: Maximum number of bytes to receive. Returns: bytes: Data received from the connection. """ if self._sock is not None: return self._sock.recv(count) else: return os.read(self.proc.stdout.fileno(), count) def close(self): """Close the SSH subprocess connection. Delegates to _close_ssh_proc to handle cleanup of the subprocess and any associated sockets or pipes. """ _close_ssh_proc(self.proc, self._sock) def get_sock_or_pipes(self): """Get the underlying I/O objects for this connection. Returns: tuple: A (kind, io_object) pair where: - If using socketpair: ('socket', socket_object) - If using pipes: ('pipes', (stdout_pipe, stdin_pipe)) """ if self._sock is not None: return "socket", self._sock else: return "pipes", (self.proc.stdout, self.proc.stdin) dromedary-0.1.1/ssh/paramiko.py000064400000000000000000000334521046102023000145720ustar 00000000000000# Copyright (C) 2006-2011 Robey Pointer # Copyright (C) 2005, 2006, 2007 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """SSH transport implementation using Paramiko library. This module provides SSH connection functionality using the Paramiko library for secure transport operations. It handles SSH authentication including agent keys, private key files, and password authentication, as well as host key verification and management. """ import getpass import logging import os from binascii import hexlify import paramiko from dromedary import _bedding, _config, _ui from dromedary.errors import TransportError from dromedary.osutils import pathjoin from dromedary.ssh import SSHConnection, SSHVendor logger = logging.getLogger("dromedary.ssh.paramiko") SYSTEM_HOSTKEYS: dict[str, dict[str, str]] = {} BRZ_HOSTKEYS: dict[str, dict[str, str]] = {} def _paramiko_auth(username, password, host, port, paramiko_transport): """Authenticate to an SSH server using paramiko. Attempts authentication in the following order: 1. SSH agent keys 2. Private key files (id_rsa, id_dsa) 3. Password authentication Args: username: SSH username, or None to use local username password: SSH password, or None for no password host: SSH hostname port: SSH port number paramiko_transport: Paramiko Transport object for the connection Raises: ConnectionError: If authentication fails or is not supported """ # paramiko requires a username, but it might be none if nothing was # supplied. If so, use the local username. if username is None: username = _config.get_auth_user( "ssh", host, port=port, default=getpass.getuser() ) agent = paramiko.Agent() for key in agent.get_keys(): logger.debug("Trying SSH agent key %s", hexlify(key.get_fingerprint()).upper()) try: paramiko_transport.auth_publickey(username, key) return except paramiko.SSHException: pass # okay, try finding id_rsa or id_dss? (posix only) if _try_pkey_auth(paramiko_transport, paramiko.RSAKey, username, "id_rsa"): return # DSSKey was removed in paramiko 4.0.0 as DSA keys are deprecated if hasattr(paramiko, "DSSKey"): if _try_pkey_auth(paramiko_transport, paramiko.DSSKey, username, "id_dsa"): return # If we have gotten this far, we are about to try for passwords, do an # auth_none check to see if it is even supported. supported_auth_types = [] try: # Note that with paramiko <1.7.5 this logs an INFO message: # Authentication type (none) not permitted. # So we explicitly disable the logging level for this action old_level = paramiko_transport.logger.level paramiko_transport.logger.setLevel(logging.WARNING) try: paramiko_transport.auth_none(username) finally: paramiko_transport.logger.setLevel(old_level) except paramiko.BadAuthenticationType as e: # Supported methods are in the exception supported_auth_types = e.allowed_types except paramiko.SSHException: # Don't know what happened, but just ignore it pass # We treat 'keyboard-interactive' and 'password' auth methods identically, # because Paramiko's auth_password method will automatically try # 'keyboard-interactive' auth (using the password as the response) if # 'password' auth is not available. Apparently some Debian and Gentoo # OpenSSH servers require this. # XXX: It's possible for a server to require keyboard-interactive auth that # requires something other than a single password, but we currently don't # support that. if ( "password" not in supported_auth_types and "keyboard-interactive" not in supported_auth_types ): raise ConnectionError( "Unable to authenticate to SSH host as" f"\n {username}@{host}\nsupported auth types: {supported_auth_types}" ) if password: try: paramiko_transport.auth_password(username, password) return except paramiko.SSHException: pass # give up and ask for a password password = _config.get_auth_password("ssh", host, username, port=port) # get_password can still return None, which means we should not prompt if password is not None: try: paramiko_transport.auth_password(username, password) except paramiko.SSHException as e: raise ConnectionError( f"Unable to authenticate to SSH host as\n {username}@{host}\n", e ) from e else: raise ConnectionError( f"Unable to authenticate to SSH host as {username}@{host}" ) def _try_pkey_auth(paramiko_transport, pkey_class, username, filename): """Attempt public key authentication with a private key file. Args: paramiko_transport: Paramiko Transport object for the connection pkey_class: Paramiko private key class (e.g., RSAKey, DSSKey) username: SSH username for authentication filename: Name of the private key file (relative to ~/.ssh/) Returns: bool: True if authentication succeeded, False otherwise """ filename = os.path.expanduser("~/.ssh/" + filename) try: key = pkey_class.from_private_key_file(filename) paramiko_transport.auth_publickey(username, key) return True except paramiko.PasswordRequiredException: password = _ui.get_password( prompt="SSH %(filename)s password", filename=os.fsdecode(filename) ) try: key = pkey_class.from_private_key_file(filename, password) paramiko_transport.auth_publickey(username, key) return True except paramiko.SSHException: logger.debug( f"SSH authentication via {os.path.basename(filename)} key failed." ) except paramiko.SSHException: logger.debug( "SSH authentication via %s key failed.", os.path.basename(filename) ) except OSError: pass return False def _ssh_host_keys_config_dir(): """Get the path to the SSH host keys configuration directory. Returns: str: Path to the directory where SSH host keys are stored """ return pathjoin(_bedding.config_dir(), "ssh_host_keys") def load_host_keys(): """Load system host keys (probably doesn't work on windows) and any "discovered" keys from previous sessions. """ global SYSTEM_HOSTKEYS, BRZ_HOSTKEYS try: SYSTEM_HOSTKEYS = paramiko.util.load_host_keys( os.path.expanduser("~/.ssh/known_hosts") ) except OSError as e: logger.debug("failed to load system host keys: %s", e) brz_hostkey_path = _ssh_host_keys_config_dir() try: BRZ_HOSTKEYS = paramiko.util.load_host_keys(brz_hostkey_path) except OSError as e: logger.debug("failed to load brz host keys: %s", e) save_host_keys() def save_host_keys(): """Save "discovered" host keys in $(config)/ssh_host_keys/.""" global SYSTEM_HOSTKEYS, BRZ_HOSTKEYS bzr_hostkey_path = _ssh_host_keys_config_dir() _bedding.ensure_config_dir_exists() try: with open(bzr_hostkey_path, "w") as f: f.write("# SSH host keys collected by bzr\n") for hostname, keys in BRZ_HOSTKEYS.items(): for keytype, key in keys.items(): f.write(f"{hostname} {keytype} {key.get_base64()}\n") except OSError as e: logger.debug("failed to save bzr host keys: %s", e) class ParamikoVendor(SSHVendor): """Vendor that uses paramiko.""" def _hexify(self, s): """Convert a byte string to uppercase hexadecimal representation. Args: s: Byte string to convert Returns: str: Uppercase hexadecimal representation of the input """ return hexlify(s).upper() def _connect(self, username, password, host, port): """Establish a low-level SSH connection using paramiko. Handles host key verification by checking against system known_hosts and breezy's stored host keys. New host keys are automatically stored. Args: username: SSH username password: SSH password or None host: SSH hostname port: SSH port number or None for default Returns: paramiko.Transport: Authenticated paramiko Transport object Raises: TransportError: If host key verification fails ConnectionError: If connection or authentication fails """ global SYSTEM_HOSTKEYS, BRZ_HOSTKEYS from dromedary.ssh.paramiko import ( _paramiko_auth, _ssh_host_keys_config_dir, load_host_keys, save_host_keys, ) load_host_keys() try: t = paramiko.Transport((host, port or 22)) t.set_log_channel("bzr.paramiko") t.start_client() except (paramiko.SSHException, OSError) as e: self._raise_connection_error(host, port=port, orig_error=e) server_key = t.get_remote_server_key() server_key_hex = self._hexify(server_key.get_fingerprint()) keytype = server_key.get_name() if host in SYSTEM_HOSTKEYS and keytype in SYSTEM_HOSTKEYS[host]: our_server_key = SYSTEM_HOSTKEYS[host][keytype] our_server_key_hex = self._hexify(our_server_key.get_fingerprint()) elif host in BRZ_HOSTKEYS and keytype in BRZ_HOSTKEYS[host]: our_server_key = BRZ_HOSTKEYS[host][keytype] our_server_key_hex = self._hexify(our_server_key.get_fingerprint()) else: logger.warning( "Adding %s host key for %s: %s", keytype, host, server_key_hex ) add = getattr(BRZ_HOSTKEYS, "add", None) if add is not None: # paramiko >= 1.X.X BRZ_HOSTKEYS.add(host, keytype, server_key) else: BRZ_HOSTKEYS.setdefault(host, {})[keytype] = server_key our_server_key = server_key our_server_key_hex = self._hexify(our_server_key.get_fingerprint()) save_host_keys() if server_key != our_server_key: filename1 = os.path.expanduser("~/.ssh/known_hosts") filename2 = _ssh_host_keys_config_dir() raise TransportError( f"Host keys for {host} do not match! {our_server_key_hex} != {server_key_hex}", [f"Try editing {filename1} or {filename2}"], ) _paramiko_auth(username, password, host, port, t) return t def connect_sftp(self, username, password, host, port): """Connect to an SFTP server using paramiko. Args: username: SSH username password: SSH password or None host: SSH hostname port: SSH port number or None for default Returns: paramiko.SFTPClient: Connected SFTP client object Raises: ConnectionError: If connection or SFTP client creation fails """ t = self._connect(username, password, host, port) try: return t.open_sftp_client() except paramiko.SSHException as e: self._raise_connection_error( host, port=port, orig_error=e, msg="Unable to start sftp client" ) def connect_ssh(self, username, password, host, port, command): """Connect to SSH server and execute a command. Args: username: SSH username password: SSH password or None host: SSH hostname port: SSH port number or None for default command: List of command arguments to execute Returns: _ParamikoSSHConnection: SSH connection object for command execution Raises: ConnectionError: If connection or command execution fails """ t = self._connect(username, password, host, port) try: channel = t.open_session() cmdline = " ".join(command) channel.exec_command(cmdline) return _ParamikoSSHConnection(channel) except paramiko.SSHException as e: self._raise_connection_error( host, port=port, orig_error=e, msg="Unable to invoke remote bzr" ) class _ParamikoSSHConnection(SSHConnection): """An SSH connection via paramiko.""" def __init__(self, channel): """Initialize a paramiko SSH connection wrapper. Args: channel: Paramiko Channel object for the SSH session """ self.channel = channel def get_sock_or_pipes(self): """Get socket or pipe information for the SSH connection. Returns: tuple: A tuple containing ("socket", channel) where channel is the paramiko Channel object """ return ("socket", self.channel) def close(self): """Close the SSH connection channel. Returns: The result of closing the paramiko channel """ return self.channel.close() paramiko_vendor = ParamikoVendor() dromedary-0.1.1/tests/__init__.py000064400000000000000000000272121046102023000150700ustar 00000000000000# Copyright (C) 2005-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Tests for dromedary transport functionality.""" import io import logging import os import re import tempfile import unittest class TestNotApplicable( unittest.TestCase.skipException if hasattr(unittest.TestCase, "skipException") else unittest.SkipTest ): """Test is not applicable to the current situation.""" TestSkipped = unittest.SkipTest def _iter_test_cases(suite_or_case): """Yield individual TestCase leaves from a TestSuite or TestCase.""" if isinstance(suite_or_case, unittest.TestCase): yield suite_or_case return for child in suite_or_case: yield from _iter_test_cases(child) def multiply_tests(tests, scenarios, result): """Multiply tests by scenarios, adding them to result. `tests` may be a TestSuite (which can contain nested suites) or an iterable of TestCase instances; suites are flattened before fan-out. """ for test in _iter_test_cases(tests): for scenario_id, scenario_attrs in scenarios: new_test = clone_test(test, scenario_id) for name, value in scenario_attrs.items(): setattr(new_test, name, value) result.addTest(new_test) return result def clone_test(test, new_id): """Clone a test case with a new id suffix.""" new_test = test.__class__(test._testMethodName) new_test._scenario_suffix = "(" + new_id + ")" return new_test class Feature: """A feature that may or may not be available.""" def available(self): try: return self._probe() except Exception: return False def _probe(self): raise NotImplementedError class _Win32Feature(Feature): def _probe(self): import sys return sys.platform == "win32" class _ParamikoFeature(Feature): def _probe(self): import importlib.util return importlib.util.find_spec("paramiko") is not None win32_feature = _Win32Feature() paramiko = _ParamikoFeature() class _AssertHelpersMixin: """Extra assertion methods for dromedary tests.""" def assertStartsWith(self, s, prefix, msg=None): if not s.startswith(prefix): if msg is None: msg = f"{s!r} does not start with {prefix!r}" raise AssertionError(msg) def assertEndsWith(self, s, suffix, msg=None): if not s.endswith(suffix): if msg is None: msg = f"{s!r} does not end with {suffix!r}" raise AssertionError(msg) def assertRaises(self, exc_type, callable=None, *args, **kwargs): if callable is not None: try: callable(*args, **kwargs) except exc_type as e: return e else: raise AssertionError(f"{exc_type.__name__} not raised") return super().assertRaises(exc_type) def assertLength(self, expected, container): if len(container) != expected: raise AssertionError( f"Expected length {expected}, got {len(container)}: {container!r}" ) def assertListRaises(self, exc_type, callable, *args, **kwargs): """Assert that fully consuming an iterator raises the given exception.""" try: list(callable(*args, **kwargs)) except exc_type: return raise AssertionError(f"{exc_type.__name__} not raised") def assertTransportMode(self, transport, path, mode): """Assert a file's mode bits via transport.stat().""" actual_mode = transport.stat(path).st_mode & 0o777 if actual_mode != mode: raise AssertionError( f"mode mismatch for {path!r}: expected {mode:o}, got {actual_mode:o}" ) def assertEqualDiff(self, expected, actual, msg=None): """Assert two values are equal; on failure print a diff.""" if expected == actual: return if isinstance(expected, bytes) and isinstance(actual, bytes): try: expected_text = expected.decode("utf-8") actual_text = actual.decode("utf-8") except UnicodeDecodeError: raise AssertionError( f"{msg + ': ' if msg else ''}{expected!r} != {actual!r}" ) from None else: expected_text = str(expected) actual_text = str(actual) import difflib diff = "\n".join( difflib.unified_diff( expected_text.splitlines(), actual_text.splitlines(), lineterm="", fromfile="expected", tofile="actual", ) ) raise AssertionError(f"{msg + chr(10) if msg else ''}values not equal:\n{diff}") def overrideAttr(self, obj, attr_name, new_value=None): """Temporarily replace an attribute, restoring it after the test.""" old_value = getattr(obj, attr_name) if new_value is not None: setattr(obj, attr_name, new_value) self.addCleanup(setattr, obj, attr_name, old_value) def recordCalls(self, obj, attr_name): """Replace a callable with a wrapper that records calls. Returns the list of call records. Restores the original after the test. """ calls = [] orig = getattr(obj, attr_name) def recorder(*args, **kwargs): calls.append((args, kwargs)) return orig(*args, **kwargs) setattr(obj, attr_name, recorder) self.addCleanup(setattr, obj, attr_name, orig) return calls class TestCase(_AssertHelpersMixin, unittest.TestCase): """Base test case for dromedary tests with extra assertion helpers.""" def setUp(self): super().setUp() self._log_stream = io.StringIO() self._log_handler = logging.StreamHandler(self._log_stream) self._log_handler.setFormatter(logging.Formatter("%(message)s")) dromedary_logger = logging.getLogger("dromedary") dromedary_logger.addHandler(self._log_handler) dromedary_logger.setLevel(logging.DEBUG) self.addCleanup(dromedary_logger.removeHandler, self._log_handler) def get_log(self): """Return captured log output.""" return self._log_stream.getvalue() def log(self, *args): """Append a message to the captured test log.""" if len(args) == 1: msg = args[0] else: msg = args[0] % args[1:] self._log_stream.write(str(msg) + "\n") def start_server(self, server): """Start a test server, registering cleanup to stop it.""" server.start_server() self.addCleanup(server.stop_server) def requireFeature(self, feature): """Skip test if feature is not available.""" if not feature.available(): raise unittest.SkipTest(f"Feature not available: {feature!r}") def assertContainsRe(self, haystack, needle, flags=0): """Assert that a string matches a regular expression.""" if not re.search(needle, haystack, flags): raise AssertionError(f"pattern {needle!r} not found in {haystack!r}") def overrideEnv(self, name, new_value): """Temporarily override an environment variable.""" old_value = os.environ.get(name) if new_value is None: os.environ.pop(name, None) else: os.environ[name] = new_value def restore(): if old_value is None: os.environ.pop(name, None) else: os.environ[name] = old_value self.addCleanup(restore) @staticmethod def _adjust_url(base, relpath): """Get a URL for the transport, adjusted by relpath.""" if relpath is not None and relpath != ".": if not base.endswith("/"): base = base + "/" if base.startswith("./") or base.startswith("/"): base += relpath else: from dromedary import urlutils base += urlutils.escape(relpath) return base class TestCaseInTempDir(TestCase): """A test case that runs in a temporary directory. Creates a fresh temporary directory before each test and changes into it. The original directory is restored and the temporary directory is cleaned up after each test. """ def setUp(self): super().setUp() self._original_dir = os.getcwd() self._tempdir = tempfile.mkdtemp(prefix="dromedary-test-") os.chdir(self._tempdir) self.addCleanup(self._cleanup_tempdir) def _cleanup_tempdir(self): os.chdir(self._original_dir) import shutil shutil.rmtree(self._tempdir, ignore_errors=True) @property def test_dir(self): """The path to the temporary directory for this test.""" return self._tempdir def build_tree(self, shape, line_endings="binary"): """Build a test tree of local files and directories under cwd. shape is a sequence of file specifications. If the final character is '/', a directory is created. """ for name in shape: if name.endswith("/"): os.mkdir(name.rstrip("/")) else: if line_endings == "binary": end = b"\n" elif line_endings == "native": end = os.linesep.encode("ascii") else: raise ValueError(f"Invalid line ending request {line_endings!r}") content = b"contents of %s%s" % (name.encode("utf-8"), end) with open(name, "wb") as f: f.write(content) def build_tree_contents(self, entries): """Build a tree with explicit file contents under cwd.""" for name, content in entries: with open(name, "wb") as f: f.write(content) class TestCaseWithMemoryTransport(TestCase): """A test case that provides a memory transport. Provides get_transport() to obtain a memory transport for testing. """ def setUp(self): super().setUp() from dromedary.memory import MemoryServer self._memory_server = MemoryServer() self._memory_server.start_server() self.addCleanup(self._memory_server.stop_server) def get_transport(self, relpath=""): """Return a memory transport for testing.""" import dromedary base_url = self._memory_server.get_url() t = dromedary.get_transport_from_url(base_url) if relpath: t = t.clone(relpath) return t class TestCaseWithTransport(TestCaseInTempDir): """A test case that provides transport access to a temporary directory.""" def get_transport(self, relpath=""): """Return a local transport for the test's temporary directory.""" from dromedary import get_transport_from_path path = os.path.join(self._tempdir, relpath) if relpath else self._tempdir os.makedirs(path, exist_ok=True) return get_transport_from_path(path) dromedary-0.1.1/tests/http_server.py000064400000000000000000000430531046102023000156770ustar 00000000000000# Copyright (C) 2006-2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import errno import http.client as http_client import http.server as http_server import os import posixpath import random import re from urllib.parse import urlparse from dromedary import osutils, urlutils from dromedary.tests import test_server class BadWebserverPath(ValueError): def __str__(self): return "path {} is not in {}".format(*self.args) class TestingHTTPRequestHandler(http_server.SimpleHTTPRequestHandler): """Handles one request. A TestingHTTPRequestHandler is instantiated for every request received by the associated server. Note that 'request' here is inherited from the base TCPServer class, for the HTTP server it is really a connection which itself will handle one or several HTTP requests. """ # Default protocol version protocol_version = "HTTP/1.1" # The Message-like class used to parse the request headers MessageClass = http_client.HTTPMessage def setup(self): http_server.SimpleHTTPRequestHandler.setup(self) self._cwd = self.server._home_dir tcs = self.server.test_case_server if tcs.protocol_version is not None: # If the test server forced a protocol version, use it self.protocol_version = tcs.protocol_version def log_message(self, format, *args): tcs = self.server.test_case_server tcs.log( 'webserver - %s - - [%s] %s "%s" "%s"', self.address_string(), self.log_date_time_string(), format % args, self.headers.get("referer", "-"), self.headers.get("user-agent", "-"), ) def handle_one_request(self): """Handle a single HTTP request. We catch all socket errors occurring when the client close the connection early to avoid polluting the test results. """ try: self._handle_one_request() except OSError as e: # Any socket error should close the connection, but some errors are # due to the client closing early and we don't want to pollute test # results, so we raise only the others. self.close_connection = 1 if len(e.args) == 0 or e.args[0] not in ( errno.EPIPE, errno.ECONNRESET, errno.ECONNABORTED, errno.EBADF, ): raise error_content_type = "text/plain" error_message_format = """\ Error code: %(code)s. Message: %(message)s. """ def send_error(self, code, message=None): """Send and log an error reply. We redefine the python-provided version to be able to set a ``Content-Length`` header as some http/1.1 clients complain otherwise (see bug #568421). :param code: The HTTP error code. :param message: The explanation of the error code, Defaults to a short entry. """ if message is None: try: message = self.responses[code][0] except KeyError: message = "???" self.log_error("code %d, message %s", code, message) content = self.error_message_format % {"code": code, "message": message} self.send_response(code, message) self.send_header("Content-Type", self.error_content_type) self.send_header("Content-Length", f"{len(content)}") self.send_header("Connection", "close") self.end_headers() if self.command != "HEAD" and code >= 200 and code not in (204, 304): self.wfile.write(content.encode("utf-8")) def _handle_one_request(self): http_server.SimpleHTTPRequestHandler.handle_one_request(self) _range_regexp = re.compile(r"^(?P\d+)-(?P\d+)?$") _tail_regexp = re.compile(r"^-(?P\d+)$") def _parse_ranges(self, ranges_header, file_size): """Parse the range header value and returns ranges. RFC2616 14.35 says that syntactically invalid range specifiers MUST be ignored. In that case, we return None instead of a range list. :param ranges_header: The 'Range' header value. :param file_size: The size of the requested file. :return: A list of (start, end) tuples or None if some invalid range specifier is encountered. """ if not ranges_header.startswith("bytes="): # Syntactically invalid header return None tail = None ranges = [] ranges_header = ranges_header[len("bytes=") :] for range_str in ranges_header.split(","): range_match = self._range_regexp.match(range_str) if range_match is not None: start = int(range_match.group("start")) end_match = range_match.group("end") if end_match is None: # RFC2616 says end is optional and default to file_size end = file_size else: end = int(end_match) if start > end: # Syntactically invalid range return None ranges.append((start, end)) else: tail_match = self._tail_regexp.match(range_str) if tail_match is not None: tail = int(tail_match.group("tail")) else: # Syntactically invalid range return None if tail is not None: # Normalize tail into ranges ranges.append((max(0, file_size - tail), file_size)) checked_ranges = [] for start, end in ranges: if start >= file_size: # RFC2616 14.35, ranges are invalid if start >= file_size return None # RFC2616 14.35, end values should be truncated # to file_size -1 if they exceed it end = min(end, file_size - 1) checked_ranges.append((start, end)) return checked_ranges def _header_line_length(self, keyword, value): header_line = f"{keyword}: {value}\r\n" return len(header_line) def send_range_content(self, file, start, length): file.seek(start) self.wfile.write(file.read(length)) def get_single_range(self, file, file_size, start, end): self.send_response(206) length = end - start + 1 self.send_header("Accept-Ranges", "bytes") self.send_header("Content-Length", "%d" % length) self.send_header("Content-Type", "application/octet-stream") self.send_header("Content-Range", "bytes %d-%d/%d" % (start, end, file_size)) self.end_headers() self.send_range_content(file, start, length) def get_multiple_ranges(self, file, file_size, ranges): self.send_response(206) self.send_header("Accept-Ranges", "bytes") boundary = "%d" % random.randint(0, 0x7FFFFFFF) # noqa: S311 self.send_header("Content-Type", f"multipart/byteranges; boundary={boundary}") boundary_line = b"--%s\r\n" % boundary.encode("ascii") # Calculate the Content-Length content_length = 0 for start, end in ranges: content_length += len(boundary_line) content_length += self._header_line_length( "Content-type", "application/octet-stream" ) content_length += self._header_line_length( "Content-Range", "bytes %d-%d/%d" % (start, end, file_size) ) content_length += len("\r\n") # end headers content_length += end - start + 1 content_length += len(boundary_line) self.send_header("Content-length", content_length) self.end_headers() # Send the multipart body for start, end in ranges: self.wfile.write(boundary_line) self.send_header("Content-type", "application/octet-stream") self.send_header( "Content-Range", "bytes %d-%d/%d" % (start, end, file_size) ) self.end_headers() self.send_range_content(file, start, end - start + 1) # Final boundary self.wfile.write(boundary_line) def do_GET(self): """Serve a GET request. Handles the Range header. """ # Update statistics self.server.test_case_server.GET_request_nb += 1 path = self.translate_path(self.path) ranges_header_value = self.headers.get("Range") if ranges_header_value is None or os.path.isdir(path): # Let the mother class handle most cases return http_server.SimpleHTTPRequestHandler.do_GET(self) try: # Always read in binary mode. Opening files in text # mode may cause newline translations, making the # actual size of the content transmitted *less* than # the content-length! f = open(path, "rb") except OSError: self.send_error(404, "File not found") return file_size = os.fstat(f.fileno())[6] ranges = self._parse_ranges(ranges_header_value, file_size) if not ranges: # RFC2616 14.16 and 14.35 says that when a server # encounters unsatisfiable range specifiers, it # SHOULD return a 416. f.close() # FIXME: We SHOULD send a Content-Range header too, # but the implementation of send_error does not # allows that. So far. self.send_error(416, "Requested range not satisfiable") return if len(ranges) == 1: (start, end) = ranges[0] self.get_single_range(f, file_size, start, end) else: self.get_multiple_ranges(f, file_size, ranges) f.close() def translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. If the server requires it, proxy the path before the usual translation """ if self.server.test_case_server.proxy_requests: # We need to act as a proxy and accept absolute urls, # which SimpleHTTPRequestHandler (parent) is not # ready for. So we just drop the protocol://host:port # part in front of the request-url (because we know # we would not forward the request to *another* # proxy). # So we do what SimpleHTTPRequestHandler.translate_path # do beginning with python 2.4.3: abandon query # parameters, scheme, host port, etc (which ensure we # provide the right behaviour on all python versions). path = urlparse(path)[2] # And now, we can apply *our* trick to proxy files path += "-proxied" return self._translate_path(path) def _translate_path(self, path): """Translate a /-separated PATH to the local filename syntax. Note that we're translating http URLs here, not file URLs. The URL root location is the server's startup directory. Components that mean special things to the local file system (e.g. drive or directory names) are ignored. (XXX They should probably be diagnosed.) Override from python standard library to stop it calling os.getcwd() """ # abandon query parameters path = urlparse(path)[2] path = posixpath.normpath(urlutils.unquote(path)) words = path.split("/") path = self._cwd for num, word in enumerate(w for w in words if w): if num == 0: _drive, word = os.path.splitdrive(word) _head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) return path class TestingHTTPServerMixin: def __init__(self, test_case_server): # test_case_server can be used to communicate between the # tests and the server (or the request handler and the # server), allowing dynamic behaviors to be defined from # the tests cases. self.test_case_server = test_case_server self._home_dir = test_case_server._home_dir class TestingHTTPServer(test_server.TestingTCPServer, TestingHTTPServerMixin): def __init__(self, server_address, request_handler_class, test_case_server): test_server.TestingTCPServer.__init__( self, server_address, request_handler_class ) TestingHTTPServerMixin.__init__(self, test_case_server) class TestingThreadingHTTPServer( test_server.TestingThreadingTCPServer, TestingHTTPServerMixin ): """A threading HTTP test server for HTTP 1.1. Since tests can initiate several concurrent connections to the same http server, we need an independent connection for each of them. We achieve that by spawning a new thread for each connection. """ def __init__(self, server_address, request_handler_class, test_case_server): test_server.TestingThreadingTCPServer.__init__( self, server_address, request_handler_class ) TestingHTTPServerMixin.__init__(self, test_case_server) class HttpServer(test_server.TestingTCPServerInAThread): """A test server for http transports. Subclasses can provide a specific request handler. """ # The real servers depending on the protocol http_server_class = { "HTTP/1.0": TestingHTTPServer, "HTTP/1.1": TestingThreadingHTTPServer, } # Whether or not we proxy the requests (see # TestingHTTPRequestHandler.translate_path). proxy_requests = False # used to form the url that connects to this server _url_protocol = "http" def __init__( self, request_handler=TestingHTTPRequestHandler, protocol_version=None ): """Constructor. :param request_handler: a class that will be instantiated to handle an http connection (one or several requests). :param protocol_version: if specified, will override the protocol version of the request handler. """ # Depending on the protocol version, we will create the approriate # server if protocol_version is None: # Use the request handler one proto_vers = request_handler.protocol_version else: # Use our own, it will be used to override the request handler # one too. proto_vers = protocol_version # Get the appropriate server class for the required protocol serv_cls = self.http_server_class.get(proto_vers, None) if serv_cls is None: raise http_client.UnknownProtocol(proto_vers) self.host = "localhost" self.port = 0 super().__init__((self.host, self.port), serv_cls, request_handler) self.protocol_version = proto_vers # Allows tests to verify number of GET requests issued self.GET_request_nb = 0 self._http_base_url = None self.logs = [] def create_server(self): return self.server_class( (self.host, self.port), self.request_handler_class, self ) def _get_remote_url(self, path): path_parts = path.split(os.path.sep) if os.path.isabs(path): if path_parts[: len(self._local_path_parts)] != self._local_path_parts: raise BadWebserverPath(path, self.test_dir) remote_path = "/".join(path_parts[len(self._local_path_parts) :]) else: remote_path = "/".join(path_parts) return self._http_base_url + remote_path def log(self, format, *args): """Capture Server log output.""" self.logs.append(format % args) def start_server(self, backing_transport_server=None): """See breezy.transport.Server.start_server. :param backing_transport_server: The transport that requests over this protocol should be forwarded to. Note that this is currently not supported for HTTP. """ # XXX: TODO: make the server back onto vfs_server rather than local # disk. if not ( backing_transport_server is None or isinstance(backing_transport_server, test_server.LocalURLServer) ): raise AssertionError( "HTTPServer currently assumes local transport, got {}".format( backing_transport_server ) ) self._home_dir = osutils.getcwd() self._local_path_parts = self._home_dir.split(os.path.sep) self.logs = [] super().start_server() self._http_base_url = f"{self._url_protocol}://{self.host}:{self.port}/" def get_url(self): """See breezy.transport.Server.get_url.""" return self._get_remote_url(self._home_dir) def get_bogus_url(self): """See breezy.transport.Server.get_bogus_url.""" # this is chosen to try to prevent trouble with proxies, weird dns, # etc return self._url_protocol + "://127.0.0.1:1/" dromedary-0.1.1/tests/https_server.py000064400000000000000000000115751046102023000160660ustar 00000000000000# Copyright (C) 2007-2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """HTTPS test server, available when ssl python module is available.""" import ssl from . import http_server, ssl_certs, test_server class TestingHTTPSServerMixin: def __init__(self, key_file, cert_file): self.key_file = key_file self.cert_file = cert_file def _get_ssl_request(self, sock, addr): """Wrap the socket with SSL.""" ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) if self.cert_file: ssl_context.load_cert_chain(self.cert_file, self.key_file) ssl_sock = ssl_context.wrap_socket( sock=sock, server_side=True, do_handshake_on_connect=False ) return ssl_sock, addr def verify_request(self, request, client_address): """Verify the request. Return True if we should proceed with this request, False if we should not even touch a single byte in the socket ! """ serving = test_server.TestingTCPServerMixin.verify_request( self, request, client_address ) if serving: try: request.do_handshake() except ssl.SSLError: # FIXME: We proabaly want more tests to capture which ssl # errors are worth reporting but mostly our tests want an https # server that works -- vila 2012-01-19 return False return serving def ignored_exceptions_during_shutdown(self, e): base = test_server.TestingTCPServerMixin return base.ignored_exceptions_during_shutdown(self, e) class TestingHTTPSServer(TestingHTTPSServerMixin, http_server.TestingHTTPServer): def __init__( self, server_address, request_handler_class, test_case_server, key_file, cert_file, ): TestingHTTPSServerMixin.__init__(self, key_file, cert_file) http_server.TestingHTTPServer.__init__( self, server_address, request_handler_class, test_case_server ) def get_request(self): sock, addr = http_server.TestingHTTPServer.get_request(self) return self._get_ssl_request(sock, addr) class TestingThreadingHTTPSServer( TestingHTTPSServerMixin, http_server.TestingThreadingHTTPServer ): def __init__( self, server_address, request_handler_class, test_case_server, key_file, cert_file, ): TestingHTTPSServerMixin.__init__(self, key_file, cert_file) http_server.TestingThreadingHTTPServer.__init__( self, server_address, request_handler_class, test_case_server ) def get_request(self): sock, addr = http_server.TestingThreadingHTTPServer.get_request(self) return self._get_ssl_request(sock, addr) class HTTPSServer(http_server.HttpServer): _url_protocol = "https" # The real servers depending on the protocol http_server_class = { "HTTP/1.0": TestingHTTPSServer, # type: ignore "HTTP/1.1": TestingThreadingHTTPSServer, # type: ignore } # Provides usable defaults since an https server requires both a # private key and a certificate to work. def __init__( self, request_handler=http_server.TestingHTTPRequestHandler, protocol_version=None, key_file=ssl_certs.build_path("server_without_pass.key"), # noqa: B008 cert_file=ssl_certs.build_path("server.crt"), # noqa: B008 ): http_server.HttpServer.__init__( self, request_handler=request_handler, protocol_version=protocol_version ) self.key_file = key_file self.cert_file = cert_file self.temp_files = [] def create_server(self): return self.server_class( (self.host, self.port), self.request_handler_class, self, self.key_file, self.cert_file, ) class HTTPSServer_urllib(HTTPSServer): """Subclass of HTTPSServer that gives https+urllib urls. This is for use in testing: connections to this server will always go through urllib where possible. """ # urls returned by this server should require the urllib client impl _url_protocol = "https+urllib" dromedary-0.1.1/tests/per_transport.py000064400000000000000000002073131046102023000162350ustar 00000000000000# Copyright (C) 2005-2011, 2015, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Tests for Transport implementations. Transport implementations tested here are supplied by TransportTestProviderAdapter. """ import contextlib import os import random import stat import sys from io import BytesIO from catalogus import pyutils import dromedary as _mod_transport from dromedary import ( ConnectedTransport, Transport, _get_transport_modules, errors, osutils, urlutils, ) from dromedary.errors import FileExists, NoSuchFile, PathError, TransportNotPossible from dromedary.memory import MemoryTransport from dromedary.osutils import getcwd from dromedary.tests import TestNotApplicable, TestSkipped, multiply_tests from .test_transport import TestTransportImplementation def get_transport_test_permutations(module): """Get the permutations module wants to have tested.""" if getattr(module, "get_test_permutations", None) is None: raise AssertionError( "transport module {} doesn't provide get_test_permutations()".format( module.__name__ ) ) return [] return module.get_test_permutations() def transport_test_permutations(): """Return a list of the klass, server_factory pairs to test.""" result = [] for module in _get_transport_modules(): try: permutations = get_transport_test_permutations( pyutils.get_named_object(module) ) for klass, server_factory in permutations: scenario = ( f"{klass.__name__},{server_factory.__name__}", {"transport_class": klass, "transport_server": server_factory}, ) result.append(scenario) except errors.DependencyNotPresent: # Continue even if a dependency prevents us # from adding this test pass return result def load_tests(loader, standard_tests, pattern): """Multiply tests for tranport implementations.""" result = loader.suiteClass() scenarios = transport_test_permutations() return multiply_tests(standard_tests, scenarios, result) class TransportTests(TestTransportImplementation): def setUp(self): super().setUp() self.overrideEnv("BRZ_NO_SMART_VFS", None) def check_transport_contents(self, content, transport, relpath): """Check that transport.get_bytes(relpath) == content.""" self.assertEqualDiff(content, transport.get_bytes(relpath)) def test_ensure_base_missing(self): """.ensure_base() should create the directory if it doesn't exist.""" t = self.get_transport() t_a = t.clone("a") self.assertFalse(t.ensure_base()) if t_a.is_readonly(): self.assertRaises(TransportNotPossible, t_a.ensure_base) return self.assertTrue(t_a.ensure_base()) self.assertTrue(t.has("a")) def test_ensure_base_exists(self): """.ensure_base() should just be happy if it already exists.""" t = self.get_transport() if t.is_readonly(): return t.mkdir("a") t_a = t.clone("a") # ensure_base returns False if it didn't create the base self.assertFalse(t_a.ensure_base()) def test_ensure_base_missing_parent(self): """.ensure_base() will fail if the parent dir doesn't exist.""" t = self.get_transport() if t.is_readonly(): return t_a = t.clone("a") t_b = t_a.clone("b") self.assertRaises(NoSuchFile, t_b.ensure_base) def test_external_url(self): """.external_url either works or raises InProcessTransport.""" t = self.get_transport() with contextlib.suppress(errors.InProcessTransport): t.external_url() def test_has(self): t = self.get_transport() files = ["a", "b", "e", "g", "%"] self.build_tree(files, transport=t) self.assertEqual(True, t.has("a")) self.assertEqual(False, t.has("c")) self.assertEqual(True, t.has(urlutils.escape("%"))) self.assertEqual(True, t.has_any(["a", "b", "c"])) self.assertEqual(False, t.has_any(["c", "d", "f", urlutils.escape("%%")])) self.assertEqual(False, t.has_any(["c", "c", "c"])) self.assertEqual(True, t.has_any(["b", "b", "b"])) def test_get(self): t = self.get_transport() content = b"contents of a\n" self.build_tree(["a"], transport=t, line_endings="binary") self.check_transport_contents(b"contents of a\n", t, "a") f = t.get("a") self.assertEqual(content, f.read()) def test_get_unknown_file(self): t = self.get_transport() files = ["a", "b"] self.build_tree(files, transport=t, line_endings="binary") self.assertRaises(NoSuchFile, t.get, "c") def iterate_and_close(func, *args): for f in func(*args): # We call f.read() here because things like paramiko actually # spawn a thread to prefetch the content, which we want to # consume before we close the handle. f.read() f.close() def test_get_directory_read_gives_ReadError(self): """Consistent errors for read() on a file returned by get().""" t = self.get_transport() if t.is_readonly(): self.build_tree(["a directory/"]) else: t.mkdir("a%20directory") # getting the file must either work or fail with a PathError try: a_file = t.get("a%20directory") except (errors.PathError, errors.RedirectRequested): # early failure return immediately. return # having got a file, read() must either work (i.e. http reading a dir # listing) or fail with ReadError with contextlib.suppress(errors.ReadError): a_file.read() def test_get_bytes(self): t = self.get_transport() files = ["a", "b", "e", "g"] contents = [ b"contents of a\n", b"contents of b\n", b"contents of e\n", b"contents of g\n", ] self.build_tree(files, transport=t, line_endings="binary") self.check_transport_contents(b"contents of a\n", t, "a") for content, fname in zip(contents, files, strict=False): self.assertEqual(content, t.get_bytes(fname)) def test_get_bytes_unknown_file(self): t = self.get_transport() self.assertRaises(NoSuchFile, t.get_bytes, "c") def test_get_bytes_with_open_write_stream_sees_all_content(self): t = self.get_transport() if t.is_readonly(): return with t.open_write_stream("foo") as handle: handle.write(b"b") self.assertEqual(b"b", t.get_bytes("foo")) with t.get("foo") as f: self.assertEqual(b"b", f.read()) def test_put_bytes(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.put_bytes, "a", b"some text for a\n" ) return t.put_bytes("a", b"some text for a\n") self.assertTrue(t.has("a")) self.check_transport_contents(b"some text for a\n", t, "a") # The contents should be overwritten t.put_bytes("a", b"new text for a\n") self.check_transport_contents(b"new text for a\n", t, "a") self.assertRaises(NoSuchFile, t.put_bytes, "path/doesnt/exist/c", b"contents") def test_put_bytes_non_atomic(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.put_bytes_non_atomic, "a", b"some text for a\n" ) return self.assertFalse(t.has("a")) t.put_bytes_non_atomic("a", b"some text for a\n") self.assertTrue(t.has("a")) self.check_transport_contents(b"some text for a\n", t, "a") # Put also replaces contents t.put_bytes_non_atomic("a", b"new\ncontents for\na\n") self.check_transport_contents(b"new\ncontents for\na\n", t, "a") # Make sure we can create another file t.put_bytes_non_atomic("d", b"contents for\nd\n") # And overwrite 'a' with empty contents t.put_bytes_non_atomic("a", b"") self.check_transport_contents(b"contents for\nd\n", t, "d") self.check_transport_contents(b"", t, "a") self.assertRaises( NoSuchFile, t.put_bytes_non_atomic, "no/such/path", b"contents\n" ) # Now test the create_parent flag self.assertRaises(NoSuchFile, t.put_bytes_non_atomic, "dir/a", b"contents\n") self.assertFalse(t.has("dir/a")) t.put_bytes_non_atomic("dir/a", b"contents for dir/a\n", create_parent_dir=True) self.check_transport_contents(b"contents for dir/a\n", t, "dir/a") # But we still get NoSuchFile if we can't make the parent dir self.assertRaises( NoSuchFile, t.put_bytes_non_atomic, "not/there/a", b"contents\n", create_parent_dir=True, ) def test_put_bytes_permissions(self): t = self.get_transport() if t.is_readonly(): return if not t._can_roundtrip_unix_modebits(): # Can't roundtrip, so no need to run this test return t.put_bytes("mode644", b"test text\n", mode=0o644) self.assertTransportMode(t, "mode644", 0o644) t.put_bytes("mode666", b"test text\n", mode=0o666) self.assertTransportMode(t, "mode666", 0o666) t.put_bytes("mode600", b"test text\n", mode=0o600) self.assertTransportMode(t, "mode600", 0o600) # Yes, you can put_bytes a file such that it becomes readonly t.put_bytes("mode400", b"test text\n", mode=0o400) self.assertTransportMode(t, "mode400", 0o400) # The default permissions should be based on the current umask umask = osutils.get_umask() t.put_bytes("nomode", b"test text\n", mode=None) self.assertTransportMode(t, "nomode", 0o666 & ~umask) def test_put_bytes_non_atomic_permissions(self): t = self.get_transport() if t.is_readonly(): return if not t._can_roundtrip_unix_modebits(): # Can't roundtrip, so no need to run this test return t.put_bytes_non_atomic("mode644", b"test text\n", mode=0o644) self.assertTransportMode(t, "mode644", 0o644) t.put_bytes_non_atomic("mode666", b"test text\n", mode=0o666) self.assertTransportMode(t, "mode666", 0o666) t.put_bytes_non_atomic("mode600", b"test text\n", mode=0o600) self.assertTransportMode(t, "mode600", 0o600) t.put_bytes_non_atomic("mode400", b"test text\n", mode=0o400) self.assertTransportMode(t, "mode400", 0o400) # The default permissions should be based on the current umask umask = osutils.get_umask() t.put_bytes_non_atomic("nomode", b"test text\n", mode=None) self.assertTransportMode(t, "nomode", 0o666 & ~umask) # We should also be able to set the mode for a parent directory # when it is created t.put_bytes_non_atomic( "dir700/mode664", b"test text\n", mode=0o664, dir_mode=0o700, create_parent_dir=True, ) self.assertTransportMode(t, "dir700", 0o700) t.put_bytes_non_atomic( "dir770/mode664", b"test text\n", mode=0o664, dir_mode=0o770, create_parent_dir=True, ) self.assertTransportMode(t, "dir770", 0o770) t.put_bytes_non_atomic( "dir777/mode664", b"test text\n", mode=0o664, dir_mode=0o777, create_parent_dir=True, ) self.assertTransportMode(t, "dir777", 0o777) def test_put_file(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.put_file, "a", BytesIO(b"some text for a\n") ) return result = t.put_file("a", BytesIO(b"some text for a\n")) # put_file returns the length of the data written self.assertEqual(16, result) self.assertTrue(t.has("a")) self.check_transport_contents(b"some text for a\n", t, "a") # Put also replaces contents result = t.put_file("a", BytesIO(b"new\ncontents for\na\n")) self.assertEqual(19, result) self.check_transport_contents(b"new\ncontents for\na\n", t, "a") self.assertRaises( NoSuchFile, t.put_file, "path/doesnt/exist/c", BytesIO(b"contents") ) def test_put_file_non_atomic(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.put_file_non_atomic, "a", BytesIO(b"some text for a\n"), ) return self.assertFalse(t.has("a")) t.put_file_non_atomic("a", BytesIO(b"some text for a\n")) self.assertTrue(t.has("a")) self.check_transport_contents(b"some text for a\n", t, "a") # Put also replaces contents t.put_file_non_atomic("a", BytesIO(b"new\ncontents for\na\n")) self.check_transport_contents(b"new\ncontents for\na\n", t, "a") # Make sure we can create another file t.put_file_non_atomic("d", BytesIO(b"contents for\nd\n")) # And overwrite 'a' with empty contents t.put_file_non_atomic("a", BytesIO(b"")) self.check_transport_contents(b"contents for\nd\n", t, "d") self.check_transport_contents(b"", t, "a") self.assertRaises( NoSuchFile, t.put_file_non_atomic, "no/such/path", BytesIO(b"contents\n") ) # Now test the create_parent flag self.assertRaises( NoSuchFile, t.put_file_non_atomic, "dir/a", BytesIO(b"contents\n") ) self.assertFalse(t.has("dir/a")) t.put_file_non_atomic( "dir/a", BytesIO(b"contents for dir/a\n"), create_parent_dir=True ) self.check_transport_contents(b"contents for dir/a\n", t, "dir/a") # But we still get NoSuchFile if we can't make the parent dir self.assertRaises( NoSuchFile, t.put_file_non_atomic, "not/there/a", BytesIO(b"contents\n"), create_parent_dir=True, ) def test_put_file_permissions(self): t = self.get_transport() if t.is_readonly(): return if not t._can_roundtrip_unix_modebits(): # Can't roundtrip, so no need to run this test return t.put_file("mode644", BytesIO(b"test text\n"), mode=0o644) self.assertTransportMode(t, "mode644", 0o644) t.put_file("mode666", BytesIO(b"test text\n"), mode=0o666) self.assertTransportMode(t, "mode666", 0o666) t.put_file("mode600", BytesIO(b"test text\n"), mode=0o600) self.assertTransportMode(t, "mode600", 0o600) # Yes, you can put a file such that it becomes readonly t.put_file("mode400", BytesIO(b"test text\n"), mode=0o400) self.assertTransportMode(t, "mode400", 0o400) # The default permissions should be based on the current umask umask = osutils.get_umask() t.put_file("nomode", BytesIO(b"test text\n"), mode=None) self.assertTransportMode(t, "nomode", 0o666 & ~umask) def test_put_file_non_atomic_permissions(self): t = self.get_transport() if t.is_readonly(): return if not t._can_roundtrip_unix_modebits(): # Can't roundtrip, so no need to run this test return t.put_file_non_atomic("mode644", BytesIO(b"test text\n"), mode=0o644) self.assertTransportMode(t, "mode644", 0o644) t.put_file_non_atomic("mode666", BytesIO(b"test text\n"), mode=0o666) self.assertTransportMode(t, "mode666", 0o666) t.put_file_non_atomic("mode600", BytesIO(b"test text\n"), mode=0o600) self.assertTransportMode(t, "mode600", 0o600) # Yes, you can put_file_non_atomic a file such that it becomes readonly t.put_file_non_atomic("mode400", BytesIO(b"test text\n"), mode=0o400) self.assertTransportMode(t, "mode400", 0o400) # The default permissions should be based on the current umask umask = osutils.get_umask() t.put_file_non_atomic("nomode", BytesIO(b"test text\n"), mode=None) self.assertTransportMode(t, "nomode", 0o666 & ~umask) # We should also be able to set the mode for a parent directory # when it is created sio = BytesIO() t.put_file_non_atomic( "dir700/mode664", sio, mode=0o664, dir_mode=0o700, create_parent_dir=True ) self.assertTransportMode(t, "dir700", 0o700) t.put_file_non_atomic( "dir770/mode664", sio, mode=0o664, dir_mode=0o770, create_parent_dir=True ) self.assertTransportMode(t, "dir770", 0o770) t.put_file_non_atomic( "dir777/mode664", sio, mode=0o664, dir_mode=0o777, create_parent_dir=True ) self.assertTransportMode(t, "dir777", 0o777) def test_put_bytes_unicode(self): t = self.get_transport() if t.is_readonly(): return unicode_string = "\u1234" self.assertRaises(TypeError, t.put_bytes, "foo", unicode_string) def test_mkdir(self): t = self.get_transport() if t.is_readonly(): # cannot mkdir on readonly transports. We're not testing for # cache coherency because cache behaviour is not currently # defined for the transport interface. self.assertRaises(TransportNotPossible, t.mkdir, ".") self.assertRaises(TransportNotPossible, t.mkdir, "new_dir") self.assertRaises(TransportNotPossible, t.mkdir, "path/doesnt/exist") return # Test mkdir t.mkdir("dir_a") self.assertEqual(t.has("dir_a"), True) self.assertEqual(t.has("dir_b"), False) t.mkdir("dir_b") self.assertEqual(t.has("dir_b"), True) self.assertEqual( [t.has(n) for n in ["dir_a", "dir_b", "dir_q", "dir_b"]], [True, True, False, True], ) # we were testing that a local mkdir followed by a transport # mkdir failed thusly, but given that we * in one process * do not # concurrently fiddle with disk dirs and then use transport to do # things, the win here seems marginal compared to the constraint on # the interface. RBC 20051227 t.mkdir("dir_g") self.assertRaises(FileExists, t.mkdir, "dir_g") # Test get/put in sub-directories t.put_bytes("dir_a/a", b"contents of dir_a/a") t.put_file("dir_b/b", BytesIO(b"contents of dir_b/b")) self.check_transport_contents(b"contents of dir_a/a", t, "dir_a/a") self.check_transport_contents(b"contents of dir_b/b", t, "dir_b/b") # mkdir of a dir with an absent parent self.assertRaises(NoSuchFile, t.mkdir, "missing/dir") def test_mkdir_permissions(self): t = self.get_transport() if t.is_readonly(): return if not t._can_roundtrip_unix_modebits(): # no sense testing on this transport return # Test mkdir with a mode t.mkdir("dmode755", mode=0o755) self.assertTransportMode(t, "dmode755", 0o755) t.mkdir("dmode555", mode=0o555) self.assertTransportMode(t, "dmode555", 0o555) t.mkdir("dmode777", mode=0o777) self.assertTransportMode(t, "dmode777", 0o777) t.mkdir("dmode700", mode=0o700) self.assertTransportMode(t, "dmode700", 0o700) t.mkdir("mdmode755", mode=0o755) self.assertTransportMode(t, "mdmode755", 0o755) # Default mode should be based on umask umask = osutils.get_umask() t.mkdir("dnomode", mode=None) self.assertTransportMode(t, "dnomode", 0o777 & ~umask) def test_opening_a_file_stream_creates_file(self): t = self.get_transport() if t.is_readonly(): return handle = t.open_write_stream("foo") try: self.assertEqual(b"", t.get_bytes("foo")) finally: handle.close() def test_opening_a_file_stream_can_set_mode(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( (TransportNotPossible, NotImplementedError), t.open_write_stream, "foo" ) return if not t._can_roundtrip_unix_modebits(): # Can't roundtrip, so no need to run this test return def check_mode(name, mode, expected): handle = t.open_write_stream(name, mode=mode) handle.close() self.assertTransportMode(t, name, expected) check_mode("mode644", 0o644, 0o644) check_mode("mode666", 0o666, 0o666) check_mode("mode600", 0o600, 0o600) # The default permissions should be based on the current umask check_mode("nomode", None, 0o666 & ~osutils.get_umask()) def test_copy_to(self): # FIXME: test: same server to same server (partly done) # same protocol two servers # and different protocols (done for now except for MemoryTransport. # - RBC 20060122 def simple_copy_files(transport_from, transport_to): files = ["a", "b", "c", "d"] self.build_tree(files, transport=transport_from) self.assertEqual(4, transport_from.copy_to(files, transport_to)) for f in files: self.check_transport_contents( transport_to.get_bytes(f), transport_from, f ) t = self.get_transport() if t.__class__.__name__ == "SFTPTransport": self.skipTest("SFTP copy_to currently too flakey to use") temp_transport = MemoryTransport("memory:///") simple_copy_files(t, temp_transport) if not t.is_readonly(): t.mkdir("copy_to_simple") t2 = t.clone("copy_to_simple") simple_copy_files(t, t2) # Test that copying into a missing directory raises # NoSuchFile if t.is_readonly(): self.build_tree(["e/", "e/f"]) else: t.mkdir("e") t.put_bytes("e/f", b"contents of e") self.assertRaises(NoSuchFile, t.copy_to, ["e/f"], temp_transport) temp_transport.mkdir("e") t.copy_to(["e/f"], temp_transport) del temp_transport temp_transport = MemoryTransport("memory:///") files = ["a", "b", "c", "d"] t.copy_to(iter(files), temp_transport) for f in files: self.check_transport_contents(temp_transport.get_bytes(f), t, f) del temp_transport for mode in (0o666, 0o644, 0o600, 0o400): temp_transport = MemoryTransport("memory:///") t.copy_to(files, temp_transport, mode=mode) for f in files: self.assertTransportMode(temp_transport, f, mode) def test_create_prefix(self): t = self.get_transport() sub = t.clone("foo").clone("bar") try: sub.create_prefix() except TransportNotPossible: self.assertTrue(t.is_readonly()) else: self.assertTrue(t.has("foo/bar")) def test_append_file(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.append_file, "a", "add\nsome\nmore\ncontents\n" ) return t.put_bytes("a", b"diff\ncontents for\na\n") t.put_bytes("b", b"contents\nfor b\n") self.assertEqual( 20, t.append_file("a", BytesIO(b"add\nsome\nmore\ncontents\n")) ) self.check_transport_contents( b"diff\ncontents for\na\nadd\nsome\nmore\ncontents\n", t, "a" ) # a file with no parent should fail.. self.assertRaises( NoSuchFile, t.append_file, "missing/path", BytesIO(b"content") ) # And we can create new files, too self.assertEqual( 0, t.append_file("c", BytesIO(b"some text\nfor a missing file\n")) ) self.check_transport_contents(b"some text\nfor a missing file\n", t, "c") def test_append_bytes(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.append_bytes, "a", b"add\nsome\nmore\ncontents\n", ) return self.assertEqual(0, t.append_bytes("a", b"diff\ncontents for\na\n")) self.assertEqual(0, t.append_bytes("b", b"contents\nfor b\n")) self.assertEqual(20, t.append_bytes("a", b"add\nsome\nmore\ncontents\n")) self.check_transport_contents( b"diff\ncontents for\na\nadd\nsome\nmore\ncontents\n", t, "a" ) # a file with no parent should fail.. self.assertRaises(NoSuchFile, t.append_bytes, "missing/path", b"content") def test_append_file_mode(self): """Check that append accepts a mode parameter.""" # check append accepts a mode t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.append_file, "f", BytesIO(b"f"), mode=None ) return t.append_file("f", BytesIO(b"f"), mode=None) def test_append_bytes_mode(self): # check append_bytes accepts a mode t = self.get_transport() if t.is_readonly(): self.assertRaises( TransportNotPossible, t.append_bytes, "f", b"f", mode=None ) return t.append_bytes("f", b"f", mode=None) def test_delete(self): # TODO: Test Transport.delete t = self.get_transport() # Not much to do with a readonly transport if t.is_readonly(): self.assertRaises(TransportNotPossible, t.delete, "missing") return t.put_bytes("a", b"a little bit of text\n") self.assertTrue(t.has("a")) t.delete("a") self.assertFalse(t.has("a")) self.assertRaises(NoSuchFile, t.delete, "a") t.put_bytes("a", b"a text\n") t.put_bytes("b", b"b text\n") t.put_bytes("c", b"c text\n") self.assertEqual([True, True, True], [t.has(n) for n in ["a", "b", "c"]]) t.delete("a") t.delete("c") self.assertEqual([False, True, False], [t.has(n) for n in ["a", "b", "c"]]) self.assertFalse(t.has("a")) self.assertTrue(t.has("b")) self.assertFalse(t.has("c")) for name in ["a", "c", "d"]: self.assertRaises(NoSuchFile, t.delete, name) # We should have deleted everything # SftpServer creates control files in the # working directory, so we can just do a # plain "listdir". # self.assertEqual([], os.listdir('.')) def test_recommended_page_size(self): """Transports recommend a page size for partial access to files.""" t = self.get_transport() self.assertIsInstance(t.recommended_page_size(), int) def test_rmdir(self): t = self.get_transport() # Not much to do with a readonly transport if t.is_readonly(): self.assertRaises(TransportNotPossible, t.rmdir, "missing") return t.mkdir("adir") t.mkdir("adir/bdir") t.rmdir("adir/bdir") # ftp may not be able to raise NoSuchFile for lack of # details when failing self.assertRaises((NoSuchFile, PathError), t.rmdir, "adir/bdir") t.rmdir("adir") self.assertRaises((NoSuchFile, PathError), t.rmdir, "adir") def test_rmdir_not_empty(self): """Deleting a non-empty directory raises an exception. sftp (and possibly others) don't give us a specific "directory not empty" exception -- we can just see that the operation failed. """ t = self.get_transport() if t.is_readonly(): return t.mkdir("adir") t.mkdir("adir/bdir") self.assertRaises(PathError, t.rmdir, "adir") def test_rmdir_empty_but_similar_prefix(self): """Rmdir does not get confused by sibling paths. A naive implementation of MemoryTransport would refuse to rmdir ".bzr/branch" if there is a ".bzr/branch-format" directory, because it uses "path.startswith(dir)" on all file paths to determine if directory is empty. """ t = self.get_transport() if t.is_readonly(): return t.mkdir("foo") t.put_bytes("foo-bar", b"") t.mkdir("foo-baz") t.rmdir("foo") self.assertRaises((NoSuchFile, PathError), t.rmdir, "foo") self.assertTrue(t.has("foo-bar")) def test_rename_dir_succeeds(self): t = self.get_transport() if t.is_readonly(): self.assertRaises( (TransportNotPossible, NotImplementedError), t.rename, "foo", "bar" ) return t.mkdir("adir") t.mkdir("adir/asubdir") t.rename("adir", "bdir") self.assertTrue(t.has("bdir/asubdir")) self.assertFalse(t.has("adir")) def test_rename_dir_nonempty(self): """Attempting to replace a nonemtpy directory should fail.""" t = self.get_transport() if t.is_readonly(): self.assertRaises( (TransportNotPossible, NotImplementedError), t.rename, "foo", "bar" ) return t.mkdir("adir") t.mkdir("adir/asubdir") t.mkdir("bdir") t.mkdir("bdir/bsubdir") # any kind of PathError would be OK, though we normally expect # DirectoryNotEmpty self.assertRaises(PathError, t.rename, "bdir", "adir") # nothing was changed so it should still be as before self.assertTrue(t.has("bdir/bsubdir")) self.assertFalse(t.has("adir/bdir")) self.assertFalse(t.has("adir/bsubdir")) def test_rename_across_subdirs(self): t = self.get_transport() if t.is_readonly(): raise TestNotApplicable("transport is readonly") t.mkdir("a") t.mkdir("b") ta = t.clone("a") tb = t.clone("b") ta.put_bytes("f", b"aoeu") ta.rename("f", "../b/f") self.assertTrue(tb.has("f")) self.assertFalse(ta.has("f")) self.assertTrue(t.has("b/f")) def test_delete_tree(self): t = self.get_transport() # Not much to do with a readonly transport if t.is_readonly(): self.assertRaises(TransportNotPossible, t.delete_tree, "missing") return # and does it like listing ? t.mkdir("adir") try: t.delete_tree("adir") except TransportNotPossible: # ok, this transport does not support delete_tree return # did it delete that trivial case? self.assertRaises(NoSuchFile, t.stat, "adir") self.build_tree( [ "adir/", "adir/file", "adir/subdir/", "adir/subdir/file", "adir/subdir2/", "adir/subdir2/file", ], transport=t, ) t.delete_tree("adir") # adir should be gone now. self.assertRaises(NoSuchFile, t.stat, "adir") def test_move(self): t = self.get_transport() if t.is_readonly(): return # TODO: I would like to use os.listdir() to # make sure there are no extra files, but SftpServer # creates control files in the working directory # perhaps all of this could be done in a subdirectory t.put_bytes("a", b"a first file\n") self.assertEqual([True, False], [t.has(n) for n in ["a", "b"]]) t.move("a", "b") self.assertTrue(t.has("b")) self.assertFalse(t.has("a")) self.check_transport_contents(b"a first file\n", t, "b") self.assertEqual([False, True], [t.has(n) for n in ["a", "b"]]) # Overwrite a file t.put_bytes("c", b"c this file\n") t.move("c", "b") self.assertFalse(t.has("c")) self.check_transport_contents(b"c this file\n", t, "b") # TODO: Try to write a test for atomicity # TODO: Test moving into a non-existent subdirectory def test_copy(self): t = self.get_transport() if t.is_readonly(): return t.put_bytes("a", b"a file\n") t.copy("a", "b") self.check_transport_contents(b"a file\n", t, "b") self.assertRaises(NoSuchFile, t.copy, "c", "d") os.mkdir("c") # What should the assert be if you try to copy a # file over a directory? # self.assertRaises(Something, t.copy, 'a', 'c') t.put_bytes("d", b"text in d\n") t.copy("d", "b") self.check_transport_contents(b"text in d\n", t, "b") def test_connection_error(self): """ConnectionError is raised when connection is impossible. The error should be raised from the first operation on the transport. """ try: url = self._server.get_bogus_url() except NotImplementedError as err: raise TestSkipped( "Transport {} has no bogus URL support.".format(self._server.__class__) ) from err t = _mod_transport.get_transport_from_url(url) self.assertRaises((errors.ConnectionError, NoSuchFile), t.get, ".bzr/branch") def test_stat(self): # TODO: Test stat, just try once, and if it throws, stop testing from stat import S_ISDIR, S_ISREG t = self.get_transport() try: st = t.stat(".") except TransportNotPossible: # This transport cannot stat return paths = ["a", "b/", "b/c", "b/d/", "b/d/e"] sizes = [14, 0, 16, 0, 18] self.build_tree(paths, transport=t, line_endings="binary") for path, size in zip(paths, sizes, strict=False): st = t.stat(path) if path.endswith("/"): self.assertTrue(S_ISDIR(st.st_mode)) # directory sizes are meaningless else: self.assertTrue(S_ISREG(st.st_mode)) self.assertEqual(size, st.st_size) self.assertRaises(NoSuchFile, t.stat, "q") self.assertRaises(NoSuchFile, t.stat, "b/a") self.build_tree(["subdir/", "subdir/file"], transport=t) subdir = t.clone("subdir") st = subdir.stat("./file") st = subdir.stat(".") def test_hardlink(self): from stat import ST_NLINK t = self.get_transport() source_name = "original_target" link_name = "target_link" self.build_tree([source_name], transport=t) try: t.hardlink(source_name, link_name) self.assertTrue(t.has(source_name)) self.assertTrue(t.has(link_name)) try: local_path = t.local_abspath(link_name) st = os.stat(local_path) self.assertEqual(st[ST_NLINK], 2) except errors.NotLocalUrl: pass except TransportNotPossible as err: raise TestSkipped( "Transport {} does not support hardlinks.".format( self._server.__class__ ) ) from err def test_symlink(self): from stat import S_ISLNK t = self.get_transport() source_name = "original_target" link_name = "target_link" self.build_tree([source_name], transport=t) try: t.symlink(source_name, link_name) self.assertTrue(t.has(source_name)) self.assertTrue(t.has(link_name)) st = t.stat(link_name) self.assertTrue( S_ISLNK(st.st_mode), f"expected symlink, got mode {st.st_mode:o}" ) except TransportNotPossible as err: raise TestSkipped( "Transport {} does not support symlinks.".format(self._server.__class__) ) from err self.assertEqual(source_name, t.readlink(link_name)) def test_readlink_nonexistent(self): t = self.get_transport() try: self.assertRaises(NoSuchFile, t.readlink, "nonexistent") except TransportNotPossible as err: raise TestSkipped( "Transport {} does not support symlinks.".format(self._server.__class__) ) from err def test_list_dir(self): # TODO: Test list_dir, just try once, and if it throws, stop testing t = self.get_transport() if not t.listable(): self.assertRaises(TransportNotPossible, t.list_dir, ".") return def sorted_list(d, transport): l = sorted(transport.list_dir(d)) return l self.assertEqual([], sorted_list(".", t)) # c2 is precisely one letter longer than c here to test that # suffixing is not confused. # a%25b checks that quoting is done consistently across transports tree_names = ["a", "a%25b", "b", "c/", "c/d", "c/e", "c2/"] if not t.is_readonly(): self.build_tree(tree_names, transport=t) else: self.build_tree(tree_names) self.assertEqual(["a", "a%2525b", "b", "c", "c2"], sorted_list("", t)) self.assertEqual(["a", "a%2525b", "b", "c", "c2"], sorted_list(".", t)) self.assertEqual(["d", "e"], sorted_list("c", t)) # Cloning the transport produces an equivalent listing self.assertEqual(["d", "e"], sorted_list("", t.clone("c"))) if not t.is_readonly(): t.delete("c/d") t.delete("b") else: os.unlink("c/d") os.unlink("b") self.assertEqual(["a", "a%2525b", "c", "c2"], sorted_list(".", t)) self.assertEqual(["e"], sorted_list("c", t)) self.assertListRaises(PathError, t.list_dir, "q") self.assertListRaises(PathError, t.list_dir, "c/f") # 'a' is a file, list_dir should raise an error self.assertListRaises(PathError, t.list_dir, "a") def test_list_dir_result_is_url_escaped(self): t = self.get_transport() if not t.listable(): raise TestSkipped("transport not listable") if not t.is_readonly(): self.build_tree(["a/", "a/%"], transport=t) else: self.build_tree(["a/", "a/%"]) names = list(t.list_dir("a")) self.assertEqual(["%25"], names) self.assertIsInstance(names[0], str) def test_clone_preserve_info(self): t1 = self.get_transport() if not isinstance(t1, ConnectedTransport): raise TestSkipped("not a connected transport") t2 = t1.clone("subdir") self.assertEqual(t1._parsed_url.scheme, t2._parsed_url.scheme) self.assertEqual(t1._parsed_url.user, t2._parsed_url.user) self.assertEqual(t1._parsed_url.password, t2._parsed_url.password) self.assertEqual(t1._parsed_url.host, t2._parsed_url.host) self.assertEqual(t1._parsed_url.port, t2._parsed_url.port) def test__reuse_for(self): t = self.get_transport() if not isinstance(t, ConnectedTransport): raise TestSkipped("not a connected transport") def new_url( scheme=None, user=None, password=None, host=None, port=None, path=None ): """Build a new url from t.base changing only parts of it. Only the parameters different from None will be changed. """ if scheme is None: scheme = t._parsed_url.scheme if user is None: user = t._parsed_url.user if password is None: password = t._parsed_url.password if user is None: user = t._parsed_url.user if host is None: host = t._parsed_url.host if port is None: port = t._parsed_url.port if path is None: path = t._parsed_url.path return str(urlutils.URL(scheme, user, password, host, port, path)) scheme = "sftp" if t._parsed_url.scheme == "ftp" else "ftp" self.assertIsNot(t, t._reuse_for(new_url(scheme=scheme))) user = "you" if t._parsed_url.user == "me" else "me" self.assertIsNot(t, t._reuse_for(new_url(user=user))) # passwords are not taken into account because: # - it makes no sense to have two different valid passwords for the # same user # - _password in ConnectedTransport is intended to collect what the # user specified from the command-line and there are cases where the # new url can contain no password (if the url was built from an # existing transport.base for example) # - password are considered part of the credentials provided at # connection creation time and as such may not be present in the url # (they may be typed by the user when prompted for example) self.assertIs(t, t._reuse_for(new_url(password="from space"))) # We will not connect, we can use a invalid host self.assertIsNot(t, t._reuse_for(new_url(host=t._parsed_url.host + "bar"))) port = 4321 if t._parsed_url.port == 1234 else 1234 self.assertIsNot(t, t._reuse_for(new_url(port=port))) # No point in trying to reuse a transport for a local URL self.assertIs(None, t._reuse_for("/valid_but_not_existing")) def test_connection_sharing(self): t = self.get_transport() if not isinstance(t, ConnectedTransport): raise TestSkipped("not a connected transport") c = t.clone("subdir") # Some transports will create the connection only when needed t.has("surely_not") # Force connection self.assertIs(t._get_connection(), c._get_connection()) # Temporary failure, we need to create a new dummy connection new_connection = None t._set_connection(new_connection) # Check that both transports use the same connection self.assertIs(new_connection, t._get_connection()) self.assertIs(new_connection, c._get_connection()) def test_reuse_connection_for_various_paths(self): t = self.get_transport() if not isinstance(t, ConnectedTransport): raise TestSkipped("not a connected transport") t.has("surely_not") # Force connection self.assertIsNot(None, t._get_connection()) subdir = t._reuse_for(t.base + "whatever/but/deep/down/the/path") self.assertIsNot(t, subdir) self.assertIs(t._get_connection(), subdir._get_connection()) home = subdir._reuse_for(t.base + "home") self.assertIs(t._get_connection(), home._get_connection()) self.assertIs(subdir._get_connection(), home._get_connection()) def test_clone(self): # TODO: Test that clone moves up and down the filesystem t1 = self.get_transport() self.build_tree(["a", "b/", "b/c"], transport=t1) self.assertTrue(t1.has("a")) self.assertTrue(t1.has("b/c")) self.assertFalse(t1.has("c")) t2 = t1.clone("b") self.assertEqual(t1.base + "b/", t2.base) self.assertTrue(t2.has("c")) self.assertFalse(t2.has("a")) t3 = t2.clone("..") self.assertTrue(t3.has("a")) self.assertFalse(t3.has("c")) self.assertFalse(t1.has("b/d")) self.assertFalse(t2.has("d")) self.assertFalse(t3.has("b/d")) if t1.is_readonly(): self.build_tree_contents([("b/d", b"newfile\n")]) else: t2.put_bytes("d", b"newfile\n") self.assertTrue(t1.has("b/d")) self.assertTrue(t2.has("d")) self.assertTrue(t3.has("b/d")) def test_clone_to_root(self): orig_transport = self.get_transport() # Repeatedly go up to a parent directory until we're at the root # directory of this transport root_transport = orig_transport new_transport = root_transport.clone("..") # as we are walking up directories, the path must be # growing less, except at the top self.assertTrue( len(new_transport.base) < len(root_transport.base) or new_transport.base == root_transport.base ) while new_transport.base != root_transport.base: root_transport = new_transport new_transport = root_transport.clone("..") # as we are walking up directories, the path must be # growing less, except at the top self.assertTrue( len(new_transport.base) < len(root_transport.base) or new_transport.base == root_transport.base ) # Cloning to "/" should take us to exactly the same location. self.assertEqual(root_transport.base, orig_transport.clone("/").base) # the abspath of "/" from the original transport should be the same # as the base at the root: self.assertEqual(orig_transport.abspath("/"), root_transport.base) # At the root, the URL must still end with / as its a directory self.assertEqual(root_transport.base[-1], "/") def test_clone_from_root(self): """At the root, cloning to a simple dir should just do string append.""" orig_transport = self.get_transport() root_transport = orig_transport.clone("/") self.assertEqual( root_transport.base + ".bzr/", root_transport.clone(".bzr").base ) def test_base_url(self): t = self.get_transport() self.assertEqual("/", t.base[-1]) def test_relpath(self): t = self.get_transport() self.assertEqual("", t.relpath(t.base)) # base ends with / self.assertEqual("", t.relpath(t.base[:-1])) # subdirs which don't exist should still give relpaths. self.assertEqual("foo", t.relpath(t.base + "foo")) # trailing slash should be the same. self.assertEqual("foo", t.relpath(t.base + "foo/")) def test_relpath_at_root(self): t = self.get_transport() # clone all the way to the top new_transport = t.clone("..") while new_transport.base != t.base: t = new_transport new_transport = t.clone("..") # we must be able to get a relpath below the root self.assertEqual("", t.relpath(t.base)) # and a deeper one should work too self.assertEqual("foo/bar", t.relpath(t.base + "foo/bar")) def test_abspath(self): # smoke test for abspath. Corner cases for backends like unix fs's # that have aliasing problems like symlinks should go in backend # specific test cases. transport = self.get_transport() self.assertEqual(transport.base + "relpath", transport.abspath("relpath")) # This should work without raising an error. transport.abspath("/") # the abspath of "/" and "/foo/.." should result in the same location self.assertEqual(transport.abspath("/"), transport.abspath("/foo/..")) self.assertEqual(transport.clone("/").abspath("foo"), transport.abspath("/foo")) # GZ 2011-01-26: Test in per_transport but not using self.get_transport? def test_win32_abspath(self): # Note: we tried to set sys.platform='win32' so we could test on # other platforms too, but then osutils does platform specific # things at import time which defeated us... if sys.platform != "win32": raise TestSkipped( "Testing drive letters in abspath implemented only for win32" ) # smoke test for abspath on win32. # a transport based on 'file:///' never fully qualifies the drive. transport = _mod_transport.get_transport_from_url("file:///") self.assertEqual(transport.abspath("/"), "file:///") # but a transport that starts with a drive spec must keep it. transport = _mod_transport.get_transport_from_url("file:///C:/") self.assertEqual(transport.abspath("/"), "file:///C:/") def test_local_abspath(self): transport = self.get_transport() try: p = transport.local_abspath(".") except (errors.NotLocalUrl, TransportNotPossible) as e: # should be formattable str(e) else: self.assertEqual(getcwd(), p) def test_abspath_at_root(self): t = self.get_transport() # clone all the way to the top new_transport = t.clone("..") while new_transport.base != t.base: t = new_transport new_transport = t.clone("..") # we must be able to get a abspath of the root when we ask for # t.abspath('..') - this due to our choice that clone('..') # should return the root from the root, combined with the desire that # the url from clone('..') and from abspath('..') should be the same. self.assertEqual(t.base, t.abspath("..")) # '' should give us the root self.assertEqual(t.base, t.abspath("")) # and a path should append to the url self.assertEqual(t.base + "foo", t.abspath("foo")) def test_iter_files_recursive(self): transport = self.get_transport() if not transport.listable(): self.assertRaises(TransportNotPossible, transport.iter_files_recursive) return self.build_tree( [ "isolated/", "isolated/dir/", "isolated/dir/foo", "isolated/dir/bar", "isolated/dir/b%25z", # make sure quoting is correct "isolated/bar", ], transport=transport, ) paths = set(transport.iter_files_recursive()) # nb the directories are not converted self.assertEqual( paths, { "isolated/dir/foo", "isolated/dir/bar", "isolated/dir/b%2525z", "isolated/bar", }, ) sub_transport = transport.clone("isolated") paths = set(sub_transport.iter_files_recursive()) self.assertEqual(paths, {"dir/foo", "dir/bar", "dir/b%2525z", "bar"}) def test_copy_tree(self): # TODO: test file contents and permissions are preserved. This test was # added just to ensure that quoting was handled correctly. # -- David Allouche 2006-08-11 transport = self.get_transport() if not transport.listable(): self.assertRaises(TransportNotPossible, transport.iter_files_recursive) return if transport.is_readonly(): return self.build_tree( [ "from/", "from/dir/", "from/dir/foo", "from/dir/bar", "from/dir/b%25z", # make sure quoting is correct "from/bar", ], transport=transport, ) transport.copy_tree("from", "to") paths = set(transport.iter_files_recursive()) self.assertEqual( paths, { "from/dir/foo", "from/dir/bar", "from/dir/b%2525z", "from/bar", "to/dir/foo", "to/dir/bar", "to/dir/b%2525z", "to/bar", }, ) def test_copy_tree_to_transport(self): transport = self.get_transport() if not transport.listable(): self.assertRaises(TransportNotPossible, transport.iter_files_recursive) return if transport.is_readonly(): return self.build_tree( [ "from/", "from/dir/", "from/dir/foo", "from/dir/bar", "from/dir/b%25z", # make sure quoting is correct "from/bar", ], transport=transport, ) from_transport = transport.clone("from") to_transport = transport.clone("to") to_transport.ensure_base() from_transport.copy_tree_to_transport(to_transport) paths = set(transport.iter_files_recursive()) self.assertEqual( paths, { "from/dir/foo", "from/dir/bar", "from/dir/b%2525z", "from/bar", "to/dir/foo", "to/dir/bar", "to/dir/b%2525z", "to/bar", }, ) def test_unicode_paths(self): """Test that we can read/write files with Unicode names.""" t = self.get_transport() # With FAT32 and certain encodings on win32 # '\xe5' and '\xe4' actually map to the same file # adding a suffix kicks in the 'preserving but insensitive' # route, and maintains the right files files = [ "\xe5.1", # a w/ circle iso-8859-1 "\xe4.2", # a w/ dots iso-8859-1 "\u017d", # Z with umlat iso-8859-2 "\u062c", # Arabic j "\u0410", # Russian A "\u65e5", # Kanji person ] no_unicode_support = getattr(self._server, "no_unicode_support", False) if no_unicode_support: self.knownFailure("test server cannot handle unicode paths") try: self.build_tree(files, transport=t, line_endings="binary") except UnicodeError as err: raise TestSkipped( "cannot handle unicode paths in current encoding" ) from err # A plain unicode string is not a valid url for fname in files: self.assertRaises(urlutils.InvalidURL, t.get, fname) for fname in files: fname_utf8 = fname.encode("utf-8") contents = b"contents of %s\n" % (fname_utf8,) self.check_transport_contents(contents, t, urlutils.escape(fname)) def test_connect_twice_is_same_content(self): # check that our server (whatever it is) is accessible reliably # via get_transport and multiple connections share content. transport = self.get_transport() if transport.is_readonly(): return transport.put_bytes("foo", b"bar") transport3 = self.get_transport() self.check_transport_contents(b"bar", transport3, "foo") # now opening at a relative url should give use a sane result: transport.mkdir("newdir") transport5 = self.get_transport("newdir") transport6 = transport5.clone("..") self.check_transport_contents(b"bar", transport6, "foo") def test_lock_write(self): """Test transport-level write locks. These are deprecated and transports may decline to support them. """ transport = self.get_transport() if transport.is_readonly(): self.assertRaises(TransportNotPossible, transport.lock_write, "foo") return transport.put_bytes("lock", b"") try: lock = transport.lock_write("lock") except TransportNotPossible: return # TODO make this consistent on all platforms: # self.assertRaises(LockError, transport.lock_write, 'lock') lock.unlock() def test_lock_read(self): """Test transport-level read locks. These are deprecated and transports may decline to support them. """ transport = self.get_transport() if transport.is_readonly(): open("lock", "w").close() else: transport.put_bytes("lock", b"") try: lock = transport.lock_read("lock") except TransportNotPossible: return # TODO make this consistent on all platforms: # self.assertRaises(LockError, transport.lock_read, 'lock') lock.unlock() def test_readv(self): transport = self.get_transport() if transport.is_readonly(): with open("a", "w") as f: f.write("0123456789") else: transport.put_bytes("a", b"0123456789") d = list(transport.readv("a", ((0, 1),))) self.assertEqual(d[0], (0, b"0")) d = list(transport.readv("a", ((0, 1), (1, 1), (3, 2), (9, 1)))) self.assertEqual(d[0], (0, b"0")) self.assertEqual(d[1], (1, b"1")) self.assertEqual(d[2], (3, b"34")) self.assertEqual(d[3], (9, b"9")) def test_readv_out_of_order(self): transport = self.get_transport() if transport.is_readonly(): with open("a", "w") as f: f.write("0123456789") else: transport.put_bytes("a", b"01234567890") d = list(transport.readv("a", ((1, 1), (9, 1), (0, 1), (3, 2)))) self.assertEqual(d[0], (1, b"1")) self.assertEqual(d[1], (9, b"9")) self.assertEqual(d[2], (0, b"0")) self.assertEqual(d[3], (3, b"34")) def test_readv_with_adjust_for_latency(self): transport = self.get_transport() # the adjust for latency flag expands the data region returned # according to a per-transport heuristic, so testing is a little # tricky as we need more data than the largest combining that our # transports do. To accomodate this we generate random data and cross # reference the returned data with the random data. To avoid doing # multiple large random byte look ups we do several tests on the same # backing data. content = random.randbytes(200 * 1024) # noqa: S311 content_size = len(content) if transport.is_readonly(): self.build_tree_contents([("a", content)]) else: transport.put_bytes("a", content) def check_result_data(result_vector): for item in result_vector: data_len = len(item[1]) self.assertEqual(content[item[0] : item[0] + data_len], item[1]) # start corner case result = list( transport.readv( "a", ((0, 30),), adjust_for_latency=True, upper_limit=content_size ) ) # we expect 1 result, from 0, to something > 30 self.assertEqual(1, len(result)) self.assertEqual(0, result[0][0]) self.assertTrue(len(result[0][1]) >= 30) check_result_data(result) # end of file corner case result = list( transport.readv( "a", ((204700, 100),), adjust_for_latency=True, upper_limit=content_size ) ) # we expect 1 result, from 204800- its length, to the end self.assertEqual(1, len(result)) data_len = len(result[0][1]) self.assertEqual(204800 - data_len, result[0][0]) self.assertTrue(data_len >= 100) check_result_data(result) # out of order ranges are made in order result = list( transport.readv( "a", ((204700, 100), (0, 50)), adjust_for_latency=True, upper_limit=content_size, ) ) # we expect 2 results, in order, start and end. self.assertEqual(2, len(result)) # start data_len = len(result[0][1]) self.assertEqual(0, result[0][0]) self.assertTrue(data_len >= 30) # end data_len = len(result[1][1]) self.assertEqual(204800 - data_len, result[1][0]) self.assertTrue(data_len >= 100) check_result_data(result) # close ranges get combined (even if out of order) for request_vector in [((400, 50), (800, 234)), ((800, 234), (400, 50))]: result = list( transport.readv( "a", request_vector, adjust_for_latency=True, upper_limit=content_size, ) ) self.assertEqual(1, len(result)) data_len = len(result[0][1]) # minimum length is from 400 to 1034 - 634 self.assertTrue(data_len >= 634) # must contain the region 400 to 1034 self.assertTrue(result[0][0] <= 400) self.assertTrue(result[0][0] + data_len >= 1034) check_result_data(result) def test_readv_with_adjust_for_latency_with_big_file(self): transport = self.get_transport() # test from observed failure case. if transport.is_readonly(): with open("a", "w") as f: f.write("a" * 1024 * 1024) else: transport.put_bytes("a", b"a" * 1024 * 1024) broken_vector = [ (465219, 800), (225221, 800), (445548, 800), (225037, 800), (221357, 800), (437077, 800), (947670, 800), (465373, 800), (947422, 800), ] results = list(transport.readv("a", broken_vector, True, 1024 * 1024)) found_items = [False] * 9 for pos, (start, length) in enumerate(broken_vector): # check the range is covered by the result for offset, data in results: if offset <= start and start + length <= offset + len(data): found_items[pos] = True self.assertEqual([True] * 9, found_items) def test_get_with_open_write_stream_sees_all_content(self): t = self.get_transport() if t.is_readonly(): return with t.open_write_stream("foo") as handle: handle.write(b"bcd") handle.flush() self.assertEqual( [(0, b"b"), (2, b"d")], list(t.readv("foo", ((0, 1), (2, 1)))) ) def test_get_smart_medium(self): """All transports must either give a smart medium, or know they can't.""" transport = self.get_transport() try: transport.get_smart_medium() except errors.NoSmartMedium: # as long as we got it we're fine pass # dromedary doesn't know about the breezy SmartClientMedium type, so # we just check that get_smart_medium either returns something or # raises NoSmartMedium. Subclasses (e.g. in breezy) override this to # also check the medium implementation type. def test_readv_short_read(self): transport = self.get_transport() if transport.is_readonly(): with open("a", "w") as f: f.write("0123456789") else: transport.put_bytes("a", b"01234567890") # This is intentionally reading off the end of the file # since we are sure that it cannot get there self.assertListRaises( ( errors.ShortReadvError, errors.InvalidRange, # Can be raised by paramiko AssertionError, ), transport.readv, "a", [(1, 1), (8, 10)], ) # This is trying to seek past the end of the file, it should # also raise a special error self.assertListRaises( (errors.ShortReadvError, errors.InvalidRange), transport.readv, "a", [(12, 2)], ) def test_no_segment_parameters(self): """Segment parameters should be stripped and stored in transport.segment_parameters. """ transport = self.get_transport("foo") self.assertEqual({}, transport.get_segment_parameters()) def test_segment_parameters(self): """Segment parameters should be stripped and stored in transport.get_segment_parameters(). """ base_url = self._server.get_url() parameters = {"key1": "val1", "key2": "val2"} url = urlutils.join_segment_parameters(base_url, parameters) transport = _mod_transport.get_transport_from_url(url) self.assertEqual(parameters, transport.get_segment_parameters()) def test_set_segment_parameters(self): """Segment parameters can be set and show up in base.""" transport = self.get_transport("foo") orig_base = transport.base transport.set_segment_parameter("arm", "board") self.assertEqual(f"{orig_base},arm=board", transport.base) self.assertEqual({"arm": "board"}, transport.get_segment_parameters()) transport.set_segment_parameter("arm", None) transport.set_segment_parameter("nonexistant", None) self.assertEqual({}, transport.get_segment_parameters()) self.assertEqual(orig_base, transport.base) def test_stat_symlink(self): # if a transport points directly to a symlink (and supports symlinks # at all) you can tell this. helps with bug 32669. t = self.get_transport() try: t.symlink("target", "link") except TransportNotPossible as err: raise TestSkipped("symlinks not supported") from err t2 = t.clone("link") st = t2.stat("") self.assertTrue(stat.S_ISLNK(st.st_mode)) def test_abspath_url_unquote_unreserved(self): """URLs from abspath should have unreserved characters unquoted. Need consistent quoting notably for tildes, see lp:842223 for more. """ t = self.get_transport() needlessly_escaped_dir = "%2D%2E%30%39%41%5A%5F%61%7A%7E/" self.assertEqual(t.base + "-.09AZ_az~", t.abspath(needlessly_escaped_dir)) def test_clone_url_unquote_unreserved(self): """Base URL of a cloned branch needs unreserved characters unquoted. Cloned transports should be prefix comparable for things like the isolation checking of tests, see lp:842223 for more. """ t1 = self.get_transport() needlessly_escaped_dir = "%2D%2E%30%39%41%5A%5F%61%7A%7E/" self.build_tree([needlessly_escaped_dir], transport=t1) t2 = t1.clone(needlessly_escaped_dir) self.assertEqual(t1.base + "-.09AZ_az~/", t2.base) def test_hook_post_connection_one(self): """Fire post_connect hook after a ConnectedTransport is first used.""" log = [] Transport.hooks.install_named_hook("post_connect", log.append, None) t = self.get_transport() self.assertEqual([], log) t.has("non-existant") # Smart-protocol transports (e.g. breezy's RemoteTransport) fire the # hook on their smart medium rather than on the transport itself. if hasattr(t, "get_smart_medium"): self.assertEqual([t.get_smart_medium()], log) elif isinstance(t, ConnectedTransport): self.assertEqual([t], log) else: self.assertEqual([], log) def test_hook_post_connection_multi(self): """Fire post_connect hook once per unshared underlying connection.""" log = [] Transport.hooks.install_named_hook("post_connect", log.append, None) t1 = self.get_transport() t2 = t1.clone(".") t3 = self.get_transport() self.assertEqual([], log) t1.has("x") t2.has("x") t3.has("x") if hasattr(t1, "get_smart_medium"): self.assertEqual([t.get_smart_medium() for t in [t1, t3]], log) elif isinstance(t1, ConnectedTransport): self.assertEqual([t1, t3], log) else: self.assertEqual([], log) dromedary-0.1.1/tests/ssl_certs/__init__.py000064400000000000000000000020741046102023000170700ustar 00000000000000# Copyright (C) 2007-2008 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """ssl_certs -- provides access to ssl keys and certificates needed by tests.""" import os # Directory containing all ssl files, keys or certificates base_dir = os.path.dirname(os.path.realpath(__file__)) def build_path(name): """Build and return a path in ssl_certs directory for name.""" return os.path.join(base_dir, name) dromedary-0.1.1/tests/ssl_certs/ca.crt000064400000000000000000000041771046102023000160620ustar 00000000000000-----BEGIN CERTIFICATE----- MIIGGDCCBACgAwIBAgIUAq4oJ9pSQG0lEE+X97sssVXPy7wwDQYJKoZIhvcNAQEL BQAwgZMxCzAJBgNVBAYTAkJaMREwDwYDVQQIDAhJbnRlcm5ldDEPMA0GA1UEBwwG QmF6YWFyMRQwEgYDVQQKDAtEaXN0cmlidXRlZDEMMAoGA1UECwwDVkNTMR8wHQYD VQQDDBZNYXN0ZXIgb2YgY2VydGlmaWNhdGVzMRswGQYJKoZIhvcNAQkBFgxjZXJ0 QG5vLnNwYW0wIBcNMjUwMTA3MTgzODM0WhgPMzAyNTAxMDcxODM4MzRaMIGTMQsw CQYDVQQGEwJCWjERMA8GA1UECAwISW50ZXJuZXQxDzANBgNVBAcMBkJhemFhcjEU MBIGA1UECgwLRGlzdHJpYnV0ZWQxDDAKBgNVBAsMA1ZDUzEfMB0GA1UEAwwWTWFz dGVyIG9mIGNlcnRpZmljYXRlczEbMBkGCSqGSIb3DQEJARYMY2VydEBuby5zcGFt MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA6M1aordgvlnDk/37VhsR JYROTKSYkfQ06qsieorN205M2dcCmugdMdjtXmGAVwE3DZOuRRx+2/4T3Y/wFMFc LCgxfFxKcy7/r6wPBuJO8h2iLijgZGFOKLfcj57nXBoSxQvHTHOuCWwMzcPepAsM kMLknZFWns+yti4Xn31cYYlWjhytp2fxP1NiUKZg6qi5CyPr112ysC61i/1brZbo IHrelbH0v/7IBsqGcSBpRDHLKQbHNBVr2AigDgdt1ayCjOuanmdCGrE7OeHd2ASS VLFpmKRpmfUjwCPPAkXJEifCmVMzO/fxXNV237qQa/osHNSz9k7cmK5U5iR2bVBR oCqae2Y+NQYgiYGJtW5XgfzBhHqH877ZqpMiPtnB9oGmFdXWnhtH8dIZ6YozylIl 9kET3h78SEeNtHMovu+c5rpmdMzbjMM6QEi2aDiJzFSJHikgPfJg4XtIq+OjnDw3 946xUiWsO/Nvq/18MvTDcsa3Y8nyiB7L4Hv4eqGdRUZD0OnfZR6dQIXJV2g0zBbP FCygXpfH2hBQXkG2WqC15RQm/TL5IkAVEogO96b2v1vRs9rabMqzUshWWdFep/1x NLNHPrgNTQvsmEE3LTgXZlRvr/8TGq2D2D6+8ia0UbpaIKMjUMVXQMee/Goo2hhq J0Pa8fpPujB0JC5jWz+yx8MCAwEAAaNgMF4wHQYDVR0OBBYEFMSNRx8FugB7FSR/ u+o0w0PvFqZAMB8GA1UdIwQYMBaAFMSNRx8FugB7FSR/u+o0w0PvFqZAMA8GA1Ud EwEB/wQFMAMBAf8wCwYDVR0PBAQDAgIEMA0GCSqGSIb3DQEBCwUAA4ICAQBZRAVE Y/J4j4fXttPzJ6g5NfemdUemBwnAW3P05AwpD4vxxb6ND5YgjfZun3hMC/Mx64BH 6jox7xCm8KfZYhQkP1xUH2Hz+wjEvdTCVEiO+psiVeyD3PF3z1OwUc/+FJ1rRulK eOIk0xUpBJI1hVcjqGqQiD3nalUgz7WDAdi6hzemxLKdVg+VN6+Td1Ue6+H8IOEI J8c5721PTnXDy8EMbr24bd+8lLw/UN4T8E8CxyJ7VxuMThuKgeNmUQlcPdCIOCy+ ge4GV16oD4xbx7ErQTYTIVJ/SmhbmRsJrrdWWTHVJWEAUkgRqy8t4y7ukLpM6fvZ wOR+pqRFGGQ7uxCMBJJ8vtYMxnloUCB2GFZ1em8QsMVdLn3ofzas3gNk47peHeZf +cFtzcwdcOJZYtf7xqdkbHkYX8j5szEkTkNaDynnPGffWYoKP/2AkbwtZ2OmhxEY PyFfk8D9/du1DTh92LCwJLzKbWRYJcSEI4FdF6z5OkVfg9QlhF/147cfVEtDd/6y ZDh3GjTX/VzyM/N4tR6VHyx75ZZRRanS6bmQAavxvUvcCMjSXWUpST5BFqctl+dV rjrhf1eaEk0GLcOsSbBHQNuY2bpBcQNN0hxbKkMMjKUksUuiNfyKWTTQQiu4Q1Ma 5guXUID4iq8fXeBX7bCuAYlEJuLaq5Xy7ps9qQ== -----END CERTIFICATE----- dromedary-0.1.1/tests/ssl_certs/ca.key000064400000000000000000000065661046102023000160660ustar 00000000000000-----BEGIN ENCRYPTED PRIVATE KEY----- MIIJtTBfBgkqhkiG9w0BBQ0wUjAxBgkqhkiG9w0BBQwwJAQQzg7AaifFy9RB75bv QUXp0wICCAAwDAYIKoZIhvcNAgkFADAdBglghkgBZQMEASoEEBbnNep+doK+koY3 PbImmuUEgglQNBgkn4ysAiQdlp7tx1e2owVxeKmNdjeUIejonvRHXdy0kFEi1hev huKf4UxrOn1+8XxmS6xshK03/9CJQp8deHTJfSwWKc/oASBODFD6Thy9Y5URNvlF vtzUwupsHlQfBN2U/PnHnL0cizW/UtYCK43t1etCL9WYQAlW3bi+Kb+xuPX2alwN OeWvn6b9AsrFIH5kq5lwcojYfCGbHDm4B17ko3wHfdZSattmBwnmiCwYTw6b1WiV AHvcpZ15IItyBdD7/2Rvo83cTvKt1Gu4tPz55PZs11SR138G/QsvCvpQtg89FrcT PtZed+OSITiTxpDcOd4xdCMi5+sAQUJR1fOCKO78Nuc5xqbWOs0M7NFIshHFWJgQ BytqGnEuQ70fw9ZY4COLoemznkxhXwTjksMVywjnk9QwfXTTz/04W0ib1Yc3TlLW LC4BVbJk/BkJLer6QbaDX8OU7D/TkDdp3Gj8oDxWatWEdXazvZvaJuCS08nh6SDM Vi7enBm9ji/apI78qisV8KRjV0QzppVfgtSTpwzOGjYq0aDxBSc4qD+gAxDvP7dY APrX6ZU7Evf7TaTNI7Xpq6QoP83Pwvu0i0TxriyyNRM/fajZo8DpSCCArMWozWQj lg2IFWx71jcxzvypKUD78ZndtQunTVWN4osjrtf6DgSeILQq3n3ixz/DQTozSXPp 2vUgG5iddmN/BgXLr3HWPn/SmuMLLs66YD8DJS094BVOv1WXtqkO2V9lds8j1pTD 548E7Qv5j3MxptTCdm4b2BEbPRGjE9Wdkd20RuTMddUFfNkqbKpMBaFXb6I2fcUl f1NlNnNQdCOFC1qLa2QcKE6cszowoKN8amLXWZ5dQCkR3dOXTqNIrWH7je4BMUxu it19mRhiOrA7vHHcwVSelWu8pjTW/5VWJdSCGMLbQTPeHGce7TBKLVFxGb5tNDDO 4hckLi2XaUkT4dBQubub5Zb55y3KeCzbu0OZW1EMgAreoUt2Msnp+RcJJ0dhyzBr pMyULSvoi/RekSnrRbH4s1q1+agzv3V2tGYqB2Phh2E0tdWcYCzR8V/HpsTJcENW fHSKfkmGQdPlGyjfKpPmPRJ4nSFh0h8lhgh3e18Ydz/4BE1JeZ4wGvQmeM1X+Zwe fuiypTTf0RjFdyBgDvAFFKpXi/2eiunJJP8SIC2hDWwsYbiB50uy7k/wXu0Rbp+7 ormZtfb/E7JvAZkbm/W5KMrPIigbwJoi5l6AD4ZsX+OCNWWhGuFy8z9alWiTUKhA Y5RY1HlENei2z56sxNWoyx4wMtpzmMxly5S0s2aGa6cXvmgW0QJCKQpguxL1AVnS LLtpkl83lc812brlSoGuWpMSIL8j+IJgNK62trlqeqEbVIylbt/TqD+MHugI5RgU wp3cUHeYRSKfonfdIyK5QazbBewLTqYRn+owQEGEQn2aJAkcdBpemZadK+YFR1oM uRiXQA0eunQHk6fzJazDLK48nViBPGZDhFFxBsP/zaVSldlMYDydDqRhKimwPP/N x2Po4k7xR+xiWzxu8pXTIRI3gccbCc9V+xZ2HAVyuAupK95lBckICS/qqDJfS/y9 wVTmevWKRKJIJRogvlj/0LCPl0sZmXoQc9UX8PpqXou0xETS6+gMojnqvvHPZecD wKPy+/64+nPySEpBAPy0LOmoxKhMXuSkyiGr+6gXNbpiKuudTNWoif2iVeyeCdWP dJ3BFJIobgLiXM47bvx/k7oGmqlpIX1VqXqJH5XjOQyw4FFmeL+VzkAVlaXzWN29 qIdypDXJ3HkQtlUp8gn//qycStHzwubavjG5TPRODM66QOoWtw95WzxpTcnwhhnS 4DPs+9B82euGBzalrvh+4kbJDog5gF9Gan7+7B9HZdRbnToU9MJVW10LOxTtIsHQ ebxHNtWEDr2aT/oADcnPQL2f8vpj721bT/G5husfDv0XdAPkDoioPNt7BvT3AZW0 dnuAmeTCKXONyxacHEeVTxWpjuuwa9nGNT/KJM5FB3vZKcvkZzFXoMPBBk8Rrfrd H73lTZYUsCgoX6RlNos/4qqBfcMi1JHQLswP5cD3Fwu4VxZYgrcdEVwE5ry3Jom5 9Yd1cTE+kHZeRLAxKK1e0vTZidPShjX6fI2PSxlyln4Xq8hzQrm9R1RpCsLgRe6H /yv42G2DCKuhTpnD63XZ4rwq0LvRuI8P5AYgcZR8WFXJ4GKJ6SpR5k/D+kABsvh4 3pn4NzTqtGw/euIWM1zp6AdyB5wFNu0L452Ppc+lCX8tB+OFUlSP8Yu+91oC+RET scZqchqxpL48vpQkN7XKqJEt0WYegNDElvoXvOvcPvvsJJl2dagg2wiwFFw+iRc0 ES3HypT0xBIRaTPfCsx7hEoXQrXh5R/gY0JKt8WEMRjrtAfz/bsZt8kc6pgGbwvO ZCCXQqThjDTdHOWh/GUI60pj7q7xmW3v4nDsANheXJXOAfRn8NC6hiYNICgeQWj9 MZYIHGR/VsQRi0msamCoAopeSuApi7yViYX0nVneJQLpwJIUMse5wPeC4dVcLVQF oXJGghXN3JNWUzDoOBMMpAPdFfAHUa8idwsnTxxY4KIORBJGksWB8Tyj0pqwBk8S udI//Vo32HoKckDQM9UG/LHYgrgxyzJTK1fYjU0Jl1MylhWNtHGK83iG5blitObQ 3L8CJY0lHSpE+wGT7CYgFIw8I3HVS9MFr13J+YTP7ld4fQpZk6W+7JsPf7UEfTvM mhRSzVmsC/e6txuTYMVYYA3grZzCNrEomijWbexlu5zGHmXndH69sNNxVAF1BrJZ rKyqMhCYqsexkGMo8zrJspeWhrFRQNGDtAOaS9iRHLM5nJDEsjje9+TIrHatd4sT Os+NL9ac9XbSXFsVUiOKIrEKOrmW+wIjDI9QKoWGBkzNes2j/OHNIGfsp+s1YL3y cpnRYnv0ZfgdAHQu1pfQ1TkJ74M7UHUIrfsCIRie9YQmWbcbZ6s5EVPSRmuHsw4U PaJMZFKQJmiNdMbUf07QbDDHiZ9/HG9tzGjuBRwTgv86uQTce/4F9TiS+Oe8ehn9 SYJubN0GvYg8k5xK7WNsntJDfQwc4Qul2N/HefT6aG7J4e9WK8dsPMtlZPkAyuKU 2Z6LLnaliXWGV1GINjfRbGrprTbMDe5orqGPcpMCATb+Bmus1k0EkDc= -----END ENCRYPTED PRIVATE KEY----- dromedary-0.1.1/tests/ssl_certs/create_ssls.py000075500000000000000000000234601046102023000176450ustar 00000000000000#! /usr/bin/env python3 # Copyright (C) 2007, 2008, 2009, 2017 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """create_ssls.py -- create ssl keys and certificates for tests. The https server requires at least a key and a certificate to start. SSL keys and certificates are created with openssl which may not be available everywhere we want to run the test suite. To simplify test writing, the necessary keys and certificates are generated by this script and used by the tests. Since creating these test keys and certificates requires a good knowledge of openssl and a lot of typing, we record all the needed parameters here. Since this will be used rarely, no effort has been made to handle exotic errors, the basic policy is that openssl should be available in the path and the parameters should be correct, any error will abort the script. Feel free to enhance that. This script provides options for building any individual files or two options to build the certificate authority files (--ca) or the server files (--server). """ import optparse import os import sys from subprocess import PIPE, CalledProcessError, Popen # We want to use the right breezy: the one we are part of # FIXME: The following is correct but looks a bit ugly _dir = os.path.dirname our_bzr = _dir(_dir(_dir(_dir(os.path.realpath(__file__))))) sys.path.insert(0, our_bzr) import contextlib from dromedary.tests import ssl_certs def error(s): print(s) exit(1) def needs(request, *paths): """Errors out if the specified path does not exists.""" missing = [p for p in paths if not os.path.exists(p)] if missing: error(f"{request} needs: {','.join(missing)}") def rm_f(path): """Rm -f path.""" with contextlib.suppress(BaseException): os.unlink(path) def _openssl(args, input=None): """Execute a command in a subproces feeding stdin with the provided input. :return: (returncode, stdout, stderr) """ cmd = ["openssl"] + args proc = Popen(cmd, stdin=PIPE) (stdout, stderr) = proc.communicate(input.encode("utf-8")) if proc.returncode: # Basic error handling, all commands should succeed raise CalledProcessError(proc.returncode, cmd) return proc.returncode, stdout, stderr ssl_params = { # Passwords "server_pass": "I will protect the communications", "server_challenge_pass": "Challenge for the CA", "ca_pass": "I am the authority for the whole... localhost", # CA identity "ca_country_code": "BZ", "ca_state": "Internet", "ca_locality": "Bazaar", "ca_organization": "Distributed", "ca_section": "VCS", "ca_name": "Master of certificates", "ca_email": "cert@no.spam", # Server identity "server_country_code": "LH", "server_state": "Internet", "server_locality": "LocalHost", "server_organization": "Testing Ltd", "server_section": "https server", "server_name": "127.0.0.1", # Always accessed under that name "server_email": "https_server@localhost", "server_optional_company_name": "", } def build_ca_key(): """Generate an ssl certificate authority private key.""" key_path = ssl_certs.build_path("ca.key") rm_f(key_path) _openssl( ["genrsa", "-passout", "stdin", "-des3", "-out", key_path, "4096"], input=f"{ssl_params['ca_pass']}\n{ssl_params['ca_pass']}\n", ) def build_ca_certificate(): """Generate an ssl certificate authority private key.""" key_path = ssl_certs.build_path("ca.key") needs("Building ca.crt", key_path) cert_path = ssl_certs.build_path("ca.crt") rm_f(cert_path) _openssl( [ "req", "-passin", "stdin", "-new", "-x509", # Will need to be generated again in 1000 years -- 20210106 "-days", "365242", "-key", key_path, "-out", cert_path, ], input="{ca_pass}\n" "{ca_country_code}\n" "{ca_state}\n" "{ca_locality}\n" "{ca_organization}\n" "{ca_section}\n" "{ca_name}\n" "{ca_email}\n".format(**ssl_params), ) def build_server_key(): """Generate an ssl server private key. We generates a key with a password and then copy it without password so that a server can use it without prompting. """ key_path = ssl_certs.build_path("server_with_pass.key") rm_f(key_path) _openssl( ["genrsa", "-passout", "stdin", "-des3", "-out", key_path, "4096"], input=f"{ssl_params['server_pass']}\n{ssl_params['server_pass']}\n", ) key_nopass_path = ssl_certs.build_path("server_without_pass.key") rm_f(key_nopass_path) _openssl( ["rsa", "-passin", "stdin", "-in", key_path, "-out", key_nopass_path], input=f"{ssl_params['server_pass']}\n", ) def build_server_signing_request(): """Create a CSR (certificate signing request) to get signed by the CA.""" key_path = ssl_certs.build_path("server_with_pass.key") needs("Building server.csr", key_path) server_csr_path = ssl_certs.build_path("server.csr") rm_f(server_csr_path) _openssl( ["req", "-passin", "stdin", "-new", "-key", key_path, "-out", server_csr_path], input="{server_pass}\n" "{server_country_code}\n" "{server_state}\n" "{server_locality}\n" "{server_organization}\n" "{server_section}\n" "{server_name}\n" "{server_email}\n" "{server_challenge_pass}\n" "{server_optional_company_name}\n".format(**ssl_params), ) def sign_server_certificate(): """CA signs server csr.""" server_csr_path = ssl_certs.build_path("server.csr") ca_cert_path = ssl_certs.build_path("ca.crt") ca_key_path = ssl_certs.build_path("ca.key") needs("Signing server.crt", server_csr_path, ca_cert_path, ca_key_path) server_cert_path = ssl_certs.build_path("server.crt") server_ext_conf = ssl_certs.build_path("server.extensions.cnf") rm_f(server_cert_path) _openssl( [ "x509", "-req", "-passin", "stdin", # Will need to be generated again in 1000 years -- 20210106 "-days", "365242", "-in", server_csr_path, "-CA", ca_cert_path, "-CAkey", ca_key_path, "-set_serial", "01", "-extfile", server_ext_conf, "-out", server_cert_path, ], input=f"{ssl_params['ca_pass']}\n", ) def build_ssls(name, options, builders): if options is not None: for item in options: builder = builders.get(item, None) if builder is None: error(f"{item} is not a known {name}") builder() opt_parser = optparse.OptionParser(usage="usage: %prog [options]") opt_parser.set_defaults(ca=False) opt_parser.set_defaults(server=False) opt_parser.add_option( "--ca", dest="ca", action="store_true", help="Generate CA key and certificate" ) opt_parser.add_option( "--server", dest="server", action="store_true", help="Generate server key, certificate signing request and certificate", ) opt_parser.add_option( "-k", "--key", dest="keys", action="append", metavar="KEY", help="generate a new KEY (several -k options can be specified)", ) opt_parser.add_option( "-c", "--certificate", dest="certificates", action="append", metavar="CERTIFICATE", help="generate a new CERTIFICATE (several -c options can be specified)", ) opt_parser.add_option( "-r", "--sign-request", dest="signing_requests", action="append", metavar="REQUEST", help="generate a new signing REQUEST (can be repeated)", ) opt_parser.add_option( "-s", "--sign", dest="signings", action="append", metavar="SIGNING", help="generate a new SIGNING (several -s options can be specified)", ) key_builders = {"ca": build_ca_key, "server": build_server_key} certificate_builders = {"ca": build_ca_certificate} signing_request_builders = {"server": build_server_signing_request} signing_builders = {"server": sign_server_certificate} if __name__ == "__main__": (Options, args) = opt_parser.parse_args() if Options.ca or Options.server: if ( Options.keys or Options.certificates or Options.signing_requests or Options.signings ): error("--ca and --server can't be used with other options") # Handles --ca before --server so that both can be used in the same run # to generate all the files needed by the https test server if Options.ca: build_ca_key() build_ca_certificate() if Options.server: build_server_key() build_server_signing_request() sign_server_certificate() else: build_ssls("key", Options.keys, key_builders) build_ssls("certificate", Options.certificates, certificate_builders) build_ssls( "signing request", Options.signing_requests, signing_request_builders ) build_ssls("signing", Options.signings, signing_builders) dromedary-0.1.1/tests/ssl_certs/server.crt000064400000000000000000000042501046102023000167750ustar 00000000000000-----BEGIN CERTIFICATE----- MIIGNjCCBB6gAwIBAgIBATANBgkqhkiG9w0BAQsFADCBkzELMAkGA1UEBhMCQlox ETAPBgNVBAgMCEludGVybmV0MQ8wDQYDVQQHDAZCYXphYXIxFDASBgNVBAoMC0Rp c3RyaWJ1dGVkMQwwCgYDVQQLDANWQ1MxHzAdBgNVBAMMFk1hc3RlciBvZiBjZXJ0 aWZpY2F0ZXMxGzAZBgkqhkiG9w0BCQEWDGNlcnRAbm8uc3BhbTAgFw0yNTAxMDcx ODM4MzVaGA8zMDI1MDEwNzE4MzgzNVowgZwxCzAJBgNVBAYTAkxIMREwDwYDVQQI DAhJbnRlcm5ldDESMBAGA1UEBwwJTG9jYWxIb3N0MRQwEgYDVQQKDAtUZXN0aW5n IEx0ZDEVMBMGA1UECwwMaHR0cHMgc2VydmVyMRIwEAYDVQQDDAkxMjcuMC4wLjEx JTAjBgkqhkiG9w0BCQEWFmh0dHBzX3NlcnZlckBsb2NhbGhvc3QwggIiMA0GCSqG SIb3DQEBAQUAA4ICDwAwggIKAoICAQC+4nchFU83BOFSEq1v6d4DKj4I73NzbkeW V1KA7KeM1OfCz75NDR24GzwqdxP2cKaCijZMrSeYv/AqhzTYa3haL9dpKVqUWqMS 0c7iR8AlpZjE2xyNHXjDo/ng9nFLmXt9prvPHTTNG1Pv2yhekmpbDnxMJfj693DF did4Aw7oLSeWNOv0PURmyNDtL9jDTRYUiddzapVp7DhB36St21ph5znt+Bh+mBBB bPdTUktKhuoFSV/wBGv8Id3LhCOzzBgthncWZQcwioIQThcTy7WdLDhM1yF35bVd GA/TvFcxHIXn39ToivPA2GcKG2EAUUq60J0sTzGhIQLYsK22CQDF4gOl6nv9dLr5 tQ6LudpQMKyJelukd7ufqgKHrwJym6riat3HXF5NRomMWlxtSoEGWS02lS3PovFl bnVFxEP7Oirx48u2KdJEYzjgDJiwdB5T6hXQ+rAc+PMGAlvSzygg1iaiRcHxWExz rQrW3DxbZtMvlWgROsoKS3QZe39XoVOKKv3mdtGgc/ETuSntui7TxfoVA5M5ACei me/xpKFX/asaVBMiXo7+3nNfBL6P08uDhb45c2TxlZDHMyJFaumlpBxeiNWDm1dl Ne5bjaMi1oP4DL2ymHuaY1tdbXXbCeaBr/AVItvRTHDY8F5todcpkvk4CI2bxMFI l3ImEQQuMwIDAQABo4GHMIGEMAkGA1UdEwQCMAAwNwYDVR0RBDAwLocEfwAAAYcQ AAAAAAAAAAAAAAAAAAAAAYIJMTI3LjAuMC4xgglsb2NhbGhvc3QwHQYDVR0OBBYE FLwyB3dn5UcprsJChNTdi87AQBb8MB8GA1UdIwQYMBaAFMSNRx8FugB7FSR/u+o0 w0PvFqZAMA0GCSqGSIb3DQEBCwUAA4ICAQAhqP8imBosxExR7hGMoolEfvaUpoa4 1dqwKn/hk+yrIv64X5WTBLBdHhIX4FMdBSk7j03q9H/DUxmQyPLS4qdg7BJvez0I NirUZEktESe/hg4lWXX0axtqnICTzc4EJKjHgl8PT9i5wmFbmIf0db2gBPllLAv7 gH/pVDRLZVrNWbYh9Gm3W4D2SlN20p8Y5bubDtLRMiwOacilYr6gyaRpTp4dGqqu fFSDsKrFlTYt48C/4WX0CmDruOLBCWq7rH0DL3D6J2HZyR7DYLhJ1waDnsQcS04D 5NL2pT/K2ZlMq/J/KkaQBawXotZdPE1z/1bHkNI6KexXmIsCz7xn3jiuGWuDoCYU aTHyBs6xQ2i9LLdrwRxIrdin8xp5sEGGHM4KdqRBb1eLDrxfDqbtosDwdg+XRrCL l1q7oHHNq/OJCRZgWQ0Hms+tUoUXrHc+Pao3I7zhkGxBeb0ytGuBoCHzifB8KW8y 1yuoVhoBz/sU+wqSAXUpJdakV6MxDCyK3OgrrIMfZKqouGnlsuabJw4YFsMBkhuD qn/PSO8OOui7ZF9nL8IxGbsyB9rnv1qWtZ+o/ViUpgJkuhjl+vhkZQUO8y6AJFU6 AS2RSkQaEqd5808WtYK6VAsBnjD04ZCT9aU4d0u0l7qKckOEtNJgUZvWfBVAYo7c +bFRLyQw0xiVYQ== -----END CERTIFICATE----- dromedary-0.1.1/tests/ssl_certs/server.csr000064400000000000000000000034351046102023000170000ustar 00000000000000-----BEGIN CERTIFICATE REQUEST----- MIIFBzCCAu8CAQAwgZwxCzAJBgNVBAYTAkxIMREwDwYDVQQIDAhJbnRlcm5ldDES MBAGA1UEBwwJTG9jYWxIb3N0MRQwEgYDVQQKDAtUZXN0aW5nIEx0ZDEVMBMGA1UE CwwMaHR0cHMgc2VydmVyMRIwEAYDVQQDDAkxMjcuMC4wLjExJTAjBgkqhkiG9w0B CQEWFmh0dHBzX3NlcnZlckBsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4IC DwAwggIKAoICAQC+4nchFU83BOFSEq1v6d4DKj4I73NzbkeWV1KA7KeM1OfCz75N DR24GzwqdxP2cKaCijZMrSeYv/AqhzTYa3haL9dpKVqUWqMS0c7iR8AlpZjE2xyN HXjDo/ng9nFLmXt9prvPHTTNG1Pv2yhekmpbDnxMJfj693DFdid4Aw7oLSeWNOv0 PURmyNDtL9jDTRYUiddzapVp7DhB36St21ph5znt+Bh+mBBBbPdTUktKhuoFSV/w BGv8Id3LhCOzzBgthncWZQcwioIQThcTy7WdLDhM1yF35bVdGA/TvFcxHIXn39To ivPA2GcKG2EAUUq60J0sTzGhIQLYsK22CQDF4gOl6nv9dLr5tQ6LudpQMKyJeluk d7ufqgKHrwJym6riat3HXF5NRomMWlxtSoEGWS02lS3PovFlbnVFxEP7Oirx48u2 KdJEYzjgDJiwdB5T6hXQ+rAc+PMGAlvSzygg1iaiRcHxWExzrQrW3DxbZtMvlWgR OsoKS3QZe39XoVOKKv3mdtGgc/ETuSntui7TxfoVA5M5ACeime/xpKFX/asaVBMi Xo7+3nNfBL6P08uDhb45c2TxlZDHMyJFaumlpBxeiNWDm1dlNe5bjaMi1oP4DL2y mHuaY1tdbXXbCeaBr/AVItvRTHDY8F5todcpkvk4CI2bxMFIl3ImEQQuMwIDAQAB oCUwIwYJKoZIhvcNAQkHMRYMFENoYWxsZW5nZSBmb3IgdGhlIENBMA0GCSqGSIb3 DQEBCwUAA4ICAQCELaxal2iWyw4zDbgaAv2zhf4ddb+NLn19XSkI+XmXA1tyW3cS zIDAfO5LKB3hM95CffCJHUsXMr1kE6ik3EzQbrtVtaCJiCBNbxznCQFInymJ2Etc 3DofIe6EfrYh930JnPDSfpf+3D0882Vj0W4NeOcx1ApzPQ/GQ76m7MsMQnFncWdv JUa/rx1u1fcD9yvMRfaAErJ/ddOGlPXTgtqNP4rk03ifztsBjyMEpbufu6DKUdsm hIK7X/JKJtX87tYcaQd38s0b7RUbPQ2w3JKd8ypU5b/YfmkSTapraOajknoKYLuO D+CiEO+VDb8CSuUNmA5k3EJy98RohpqslwADrY7ylE74JAt/3psaFgsyKVinj5Gg 0Ne0JGYWFILNndCVeAsUkidppEaeGYtVq7pcvpVKhEkz3ruAi/XoN79+xS0OZyHx vKqmnYfErpgoZqTfiofHj86H/paMRe14lEUkmDCqJ9vBae09NhoyqOv/scECK52c /nTYkdtdPm03Oo1LoMVRnCLCjuQDF4p6QiGEDVHJBeSAATnCz77Gp4qhhtVI29Ns SOvrhs9jVHz0Ytfofv03tEuFTck94lypzFOj2/jIOa2xVntHWdgMr43C19buWnfK E6bV/GjQPGG5n2N2MQpIPiruwhW53Z3G9FvfI2RMZ6y6xWpxMf3MZU6yLQ== -----END CERTIFICATE REQUEST----- dromedary-0.1.1/tests/ssl_certs/server.extensions.cnf000064400000000000000000000003221046102023000211450ustar 00000000000000basicConstraints=CA:FALSE subjectAltName=@my_subject_alt_names subjectKeyIdentifier = hash [ my_subject_alt_names ] IP.1 = 127.0.0.1 IP.2 = ::1 DNS.1 = 127.0.0.1 # to keep python 2.7 happy DNS.2 = localhost dromedary-0.1.1/tests/ssl_certs/server_with_pass.key000064400000000000000000000065361046102023000210670ustar 00000000000000-----BEGIN ENCRYPTED PRIVATE KEY----- MIIJpDBWBgkqhkiG9w0BBQ0wSTAxBgkqhkiG9w0BBQwwJAQQn5K/WXzTg7+Xo73l BnSBuAICCAAwDAYIKoZIhvcNAgkFADAUBggqhkiG9w0DBwQIctVzZm7/GYAEgglI DiYYoOEwvKjXS/tQeSCg72NOsgMfuJPXZlRcFJdwYSQ95iBnN9iEfWyUqJO/9TPs +jBo3XGssXE7J2RHvOXJU6qmqHfnRf+H030SSCcWWR+qGWeJtknt2IMjzPtoJlqS K4Vl07trEaEdiAp2egr6R7YDab/CcN2QFcL0NFVHYxibe4VMcOGgu7XmVD2vzY5x 54FGcLSMB0U4oN/qAMC59jxJYk2jiaiGfS6/hW/Wlabq/i2AjC6LqNJyyeCBwU76 aZgkjTMAvbDCR/A+sOSv2PciS2jgff96acDGMfkm38xj41XMr8CZkRa+KpRd4aCX WWFti9N9GVFL6dKEO7uvossgPsEq2U7rDs8MdqTTfHwzfGbPsStx5mAF0xR72PqZ k/5hx9gTk2GsZ8p3wlz2Iu4qpKedKWNMtQvp9weXX2QaylFASCY2HyC8XOlxBRbT D08+6ypJ1aXELMiUU4+rkIjILMLZedr5r13md2puBP1v/rNt2XjC//NqmftYhIpX 3ppGXHYkukvZ4Y0tVphiywGUKVRmF9CdHOZ+xEUQDqU3Ea7hLxx2c/VFv5r7VpbU ZsGvOwQOi8UrPPTkwXJApENYeKw5FdMsXV9FOPbgwOAVP5D8Wq99DGOKA6t6jfwZ qj5AXkEP2gu6Gex1DHI2w4emCisRSF0Q9H+ZCEiKnIVFhawCZcPUJqPROdXbDE9C ZpsC34GK4Nq8jl3sfrvgOHN8/eoDu1bRIjhu1MjoWqMCOa0OmL2X4kFU956O+BtR Tgx8FwgQRzM4MKRfBensHsK34WK8HI2JY3Oaj0v1ygCb+9EaNaJQlJ1gU+icBh0R NlM8qkhPktXjbRVsnefBn0wKlvuMLShB78Hn3BHZaSUtwnyBRJWdvJ+CTanEThl4 tWHytvxwfFdVs/K3Lt4OQum32QOGxZ/yTXP6xZ6UE2P2JS0/gMb1VmSxBM/T8hTI zWjK5WNLKgSB5PSxnvAxO+YC+oGEqIullJ2ad6u6/k+JdOU96x2YGwmsTLaluNI9 fn1HljHjTiqNWgIS6hAtQHwnjzuNIPl5MMOGNWPcNyXm8lwl6KzC154CsE8qL2/m yfcd5XmJ8N1USKjHs3KWCLvGy5gJo8oV2rWIHqWmW52MeAWar0DDdFje5gGc/jHL Is6aUSspb3tGDJ0SYs73Og3CWTUKu4Rwouhu4XJWKVJ8pr7w/R7ofZVClBcYpGbu wC0dE55Ot9hdwQTBMObE89g9uNHfCPc4nPsZ+XQBYvfXEMgBvlJZ80Am3IAWu5NR Rdr7Q2nsb6epAn1OoT3m1My5oDIJ5WEbKaiwDTAcZPYY4ivb9i+cPMWPK4/VrDWr aqsZ9HHIwUxazj6LLfAyO6N6dJ9DTuwDyMJ9qK7ez69kbQSn3ex+oi0pPhFxgLFp lwMyPU1SS1xVi6jJDutaYDGKfTbuDMdBubSwcwz9JyT6FYR+DTowSZ/ytRrG3E1C DRCobe59B/e9VLlk3VOSmw/E7zgw1s+jMj8SYnyhNFqqT5nvgSCPV3nPDkpY2cK9 R5gDyM8Nl07wWByhsvRKYmjal+n++k5bEZmaEHnfQf4OWpwgnNKFKC2REYboUKoI bMBmTfcwzbaooJN9G3FZ6RtjWwMDQwIVFB9XaA2+s4q4kYObmANr4E5Eef3uhSz5 6Fm5i6H6nM5NrcTQEERvi52xMO/ZIWr10Hqtf2jhjCx0lxwJAn3v9IUzSyeBjMBr 2T0X4YEtvUO+PCKBUcKww6RntLC1UYjEARUyJBF8VCmFCC8rJjVLIkziyOeeYaSn FzDhh0NGD7s7m7rvhW00RZ3si63LphyMsRai0BD39lBWO59eEic16mfA8JnMXsrJ z8zSsGnxuVjIwLrAmWnhJpUwGGS1CkfmK18hW1m8io/NlDMANfx+Gxh5uMEjYehM hAgSjNLUCIbbidS6GubXe5EOIiwjMZgQ/Q2OmrPOkPoTW0b9ixBMgnkLH4X2XZQZ dq54NFvJAwbDkjSZvXtM7aaPWtskwETZgWB+BLJXI1YiGVVBJhGgYv/SL2WhkzIy VCzIHIZqdVWsEpoix5VTMRN2ItZ2b3wmgo0HhsX1Hfxoz0aCGYpW5zIfZVbvcEj8 PWH1OgaEkT84qVDa0N63r7c9HYkgym901DsdWGNkWerei9yyqdb/4pLQW2q8aXvV 2k0o5ZKN/61lXcRks8usrmDo6rcU/pRWALjjAFK2h0esYfSiQZ3pq/XhcmTNnB72 sRs/5r5eHNjfBjKGe89lhH7VrxOx36UMNxwTIsJtVjunfhHaaAx6bf14XL9VKZla RRDrHCe1ZW4Uz0xEjNI0gqcV8gxIyhfulzQsOuF5QkZ+Ewu0fnSxkcehvldXNMAu oiFSR4rNNgbJRy5AXD7uDdCLokxAMD+XSOdwT7aI+HY3CX0Z8HM6fZolcZQkqHnp OoVTAjgzOl2fSHGLpiUts3PZngzjrKp/kvdQZd4A2lXIqcmqErk7homzLqsLVU9G pOYEstyut7IMzmNm0kEOVkvJpaCy3dAO+TRvDX3AYyKSsZ6/QrPqH2w8kgAOX4mB 1BWc15bttlu91g0D61jYz7/5wMkosQ4hrEvrbNM+CB7lQf9i+1egDh58mPbGGzw2 pKrHh4vR99dh+kUehulK4GBTIYu3Ckm7gZbaahE3XhfqpYsgzp4xZUCVAm3V1IMy 9wKoMf+h5gTFB4l1oIzArwYM8CQmcNvtF7JI8wrHaOQoIs3Z6coNsnQzL0lx4/x/ 8BMC3l0x6v7thzZeeVK8UjTPoMB6eVBige7kuT6+3p5g5QEa30FMnT623FMM/GH2 Wdxpp9HQZRe5D/PemWHxWxlQmSpmIhzHtyeHAfyuXwfDovbY2L5Y0YbcXxeKD+0d xGijAcMCBz5ahGpThYoa0nCSPlnZFwP+ipybn7YD2yM0z7j59Okd7ggSQDkU2Y3L rt8vcCdXSp4EgRWJZ9NTYFFG+JVbXBxdnQIZ7YkbKygSMRj5v4OrcawNpbtW/alX qp5/U4E/lbtbh68YhpYNd0QGRpDDrcSHlHFv4gv2beEdbrt1VYkCfCfo9krO2kJT TScDXHWASw68sYjTgc6fQqAIFxN4QsndoSMUdcaUlGSu3Pn2eSzSmwdLK1saWqJv RqBFvE3FkaUmAZTvoVgGt3uyEh4SSSR5 -----END ENCRYPTED PRIVATE KEY----- dromedary-0.1.1/tests/ssl_certs/server_without_pass.key000064400000000000000000000063101046102023000216050ustar 00000000000000-----BEGIN PRIVATE KEY----- MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC+4nchFU83BOFS Eq1v6d4DKj4I73NzbkeWV1KA7KeM1OfCz75NDR24GzwqdxP2cKaCijZMrSeYv/Aq hzTYa3haL9dpKVqUWqMS0c7iR8AlpZjE2xyNHXjDo/ng9nFLmXt9prvPHTTNG1Pv 2yhekmpbDnxMJfj693DFdid4Aw7oLSeWNOv0PURmyNDtL9jDTRYUiddzapVp7DhB 36St21ph5znt+Bh+mBBBbPdTUktKhuoFSV/wBGv8Id3LhCOzzBgthncWZQcwioIQ ThcTy7WdLDhM1yF35bVdGA/TvFcxHIXn39ToivPA2GcKG2EAUUq60J0sTzGhIQLY sK22CQDF4gOl6nv9dLr5tQ6LudpQMKyJelukd7ufqgKHrwJym6riat3HXF5NRomM WlxtSoEGWS02lS3PovFlbnVFxEP7Oirx48u2KdJEYzjgDJiwdB5T6hXQ+rAc+PMG AlvSzygg1iaiRcHxWExzrQrW3DxbZtMvlWgROsoKS3QZe39XoVOKKv3mdtGgc/ET uSntui7TxfoVA5M5ACeime/xpKFX/asaVBMiXo7+3nNfBL6P08uDhb45c2TxlZDH MyJFaumlpBxeiNWDm1dlNe5bjaMi1oP4DL2ymHuaY1tdbXXbCeaBr/AVItvRTHDY 8F5todcpkvk4CI2bxMFIl3ImEQQuMwIDAQABAoICAAkI8xaPyaYTBw85bxgi+60u rK0DmHVYPO8yxubvTKbv1OB1sM441rVGJLzl0f4SKu9210cd0wf53cZFjAzKWXH7 XbjOikkHWTykzaQMPV4KzoZS0LElOfgYpNUvFQG9DAlQgQc2nK8wofJybyC60Wnp 75wzF+vZFm9iPlAB5Qy8Rmlnq9ttovUygCEZ6Kql1Wu3cok1/Eh9M9R6X3MTNN35 cdZ/rbkgPXS7UaGR/ZpTdHQ3muDjdLEEcVkWshHqkSJmgPCAa6yygaF/8LuxIrGD zE5myGDCcKktYGJnQcFDVls5TvyKxyrTk1z5GshHEMBy612TigfUZiwXgMFi/7RJ JE4Ul6vdiMxqIqrIDFZuYlRNu5HpY3JKw4e3uMPVEEJ0zX7ecKlNGHlO6d2TVYDn u/K365F7ZqkDiLi4WaWnUcl4ZXBLCVbWUPPxUxT/I1q8hcd+NDSpA+e2Ud3sv968 rsBksqSjHW/mV+INsFbLvbomI/CAdMjg/O2mcy/Qf52Pmju3/usxG1RJurMFgMkF fGn1L86edz7H2iNpWihB9Ny/lJeu5C0BIE28vwI/MrCgSd3DLUqTRPoi61ldUlnP 5Tyagsroy0CXMQREX13f6ljohlMFuAb3bdBDZxtCLDYMLlflOmbO2yi32tzpNtKd 73t9kyjqWUVcZQftzCsxAoIBAQDkatb9fWHRCURDqba5hvLKst60eqv6X3flPQ21 t3u9HV+3i1mcE7Eqc59XGFWYOa6r+UB7zwWZ4DnrwDHpHyRnlhbE2qTBPOO6yTN7 oQr+04xSO91+567P0nKUcSn19vpziR/nWHMwXFjlId2kbMAR2Q0+YLVjMFvyo3rQ gQwBA/9SwPgGIPag8FlUPZRxHsKAT4gVnth4UsfRFFg/qvHewLjsdqF2Fl/uIyeI F1mCgrcWBWGfBHOlNwotN8voS6eWTAP5y+NF/FQQvI48qVNOYYtcUsmXpBGlo5r9 6GBgl26JkN1WtF7k2ryQyMtPZXHnxa4gtsa42WcwrXe03Wg/AoIBAQDV71x3/b6y VEq9j1crpIhAxnF+a6VjVZRbeyl9IbzIwgmgrL2kmduCxK9/dJQMeX/ZCYFATK64 rHPJ3ie9PRoVzhCVYEIPe/g06knXMjbacRUt1/gbQDBUUzAMPSqyF0dZvpiyVuWA Ei/tELgOg85+CFaCGcq58wfTTS76JowGCBGYDHYYqcxRjo9kgvtaPpHT0zMsMx0q a/EkOOp6dIhxWbv7uEskkBE22U5XyM0pGUb9yyjzUfzR2jM1/UR45wSixZWBDsp0 3ivY+LzcejPxwFBkjkTvbK/naYyHCt3A9JtJqVI92jLBNr6J34PyEv1soynBqGlO aFErz7aLWV0NAoIBAQDj3d2ZxlIlueva4FzEGmbtZaGcRGB6hnDSRKT/qgqML9iD /0Um2dI8+ll0BnelQ64IK8BzgqQgzLqbgAGKgaHMoMMYINKJX9gDR6LPa2nPq3Tp uIUPi8st0dCyW24zzO4vAhXMscU/8nBQLQeydzbo1zJKDyoEyIKBvSrRBFvYS4eT o6QKYSoIhZ1n14LVko0QecbVYsCq0FI8NuKOqVdfE72nT/VlG48ZvwI51qlZ9FL3 aejoPQWtRQom7+nAVHDcE/tHYGnbMI0goSn4RCcyI0dmk8Q+PdPI/Tyqnf4/ffIs 1FKqo4ejIL9KZLXF//qw79j1E8GNOHyj5/lqehH7AoIBAH9XhTEfZz17EyoWgorF 2xzDgpb/uGiSbkat3xpO1LKjXVu4twGdW55ROS2i1OVABSvJjpgZjP78F8gXZows LLTB/fkMXQYegrXsp5tilmgcW8D4BwUhhiMLiVQfrKRpWt0+qGGve9hp+wEfrI9n Qaifie7TL2rUENpj3QylmT+V2fMpp7oyiB4bv5rSpI8pI2B1HMa4fincKqOnBVty tizSfyTspD3VS4nce9eg/Q3zr+At5+g960F2onkGkpVs3cON3Nn/Vd+Ox4bVOIX8 b6L9GF1imgHyLhqPJ0jS3QCYGT8VfJC4qvF7hptG4qFbUGI0FQzjFBvJ2Fc9wqjb vpkCggEAFU2w1j8HAvDDJ1GSiwwc353dkCcvwBUAlN+Vryf08+T9p0rLeViZ7u6n sI1otjQhNNitGEmOu30wQTH2eq8nb8UwcZjlxQxMeozKmun74lb2w0l61ZtTVsKg PyTpwRU29s0o7gcCNQQL2ZT1OnnpoFXT+h9A46Qt8pqIKtMzEQjjHiBaD2xqv7O1 UzmSCLp2Nj7+INV/zIOX8HicG7evQHj6pqSCRfPgxJKbPuBfKZJeP4d0xJ6+oapY /ePK5+qu8bpLd8JzEICVgldVYSCeKTexG5enUN9nbktj5ozGv73cYSS+x3GlxhmC qaFsUyg+Hs0Zme451SYuGP/r613TEQ== -----END PRIVATE KEY----- dromedary-0.1.1/tests/stub_sftp.py000064400000000000000000000513431046102023000153440ustar 00000000000000# Copyright (C) 2005, 2006, 2008-2011 Robey Pointer , Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """A stub SFTP server for loopback SFTP testing. Adapted from the one in paramiko's unit tests. """ import os import socket import socketserver import sys import time import paramiko from dromedary import ssh from dromedary.ssh.paramiko import ParamikoVendor from .. import osutils, trace, urlutils from . import test_server class StubServer(paramiko.ServerInterface): def __init__(self, test_case_server): paramiko.ServerInterface.__init__(self) self.log = test_case_server.log def check_auth_password(self, username, password): # all are allowed self.log(f"sftpserver - authorizing: {username}") return paramiko.AUTH_SUCCESSFUL def check_channel_request(self, kind, chanid): self.log(f"sftpserver - channel request: {kind}, {chanid}") return paramiko.OPEN_SUCCEEDED class StubSFTPHandle(paramiko.SFTPHandle): def stat(self): try: return paramiko.SFTPAttributes.from_stat(os.fstat(self.readfile.fileno())) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) def chattr(self, attr): # python doesn't have equivalents to fchown or fchmod, so we have to # use the stored filename trace.mutter("Changing permissions on %s to %s", self.filename, attr) try: paramiko.SFTPServer.set_file_attr(self.filename, attr) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) class StubSFTPServer(paramiko.SFTPServerInterface): def __init__(self, server, root, home=None): paramiko.SFTPServerInterface.__init__(self, server) # All paths are actually relative to 'root'. # this is like implementing chroot(). self.root = root if home is None: self.home = "" else: if not home.startswith(self.root): raise AssertionError( f"home must be a subdirectory of root ({home} vs {root})" ) self.home = home[len(self.root) :] if self.home.startswith("/"): self.home = self.home[1:] server.log("sftpserver - new connection") def _realpath(self, path): # paths returned from self.canonicalize() always start with # a path separator. So if 'root' is just '/', this would cause # a double slash at the beginning '//home/dir'. if self.root == "/": return self.canonicalize(path) return self.root + self.canonicalize(path) if sys.platform == "win32": def canonicalize(self, path): # Win32 sftp paths end up looking like # sftp://host@foo/h:/foo/bar # which means absolute paths look like: # /h:/foo/bar # and relative paths stay the same: # foo/bar # win32 needs to use the Unicode APIs. so we require the # paths to be utf8 (Linux just uses bytestreams) thispath = path.decode("utf8") if path.startswith("/"): # Abspath H:/foo/bar return os.path.normpath(thispath[1:]) else: return os.path.normpath(os.path.join(self.home, thispath)) else: def canonicalize(self, path): if os.path.isabs(path): return osutils.normpath(path) else: return osutils.normpath("/" + os.path.join(self.home, path)) def chattr(self, path, attr): try: paramiko.SFTPServer.set_file_attr(path, attr) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK def list_folder(self, path): path = self._realpath(path) try: out = [] # TODO: win32 incorrectly lists paths with non-ascii if path is not # unicode. However on unix the server should only deal with # bytestreams and posix.listdir does the right thing if sys.platform == "win32": flist = [f.encode("utf8") for f in os.listdir(path)] else: flist = os.listdir(path) for fname in flist: attr = paramiko.SFTPAttributes.from_stat( os.stat(osutils.pathjoin(path, fname)) ) attr.filename = fname out.append(attr) return out except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) def stat(self, path): path = self._realpath(path) try: return paramiko.SFTPAttributes.from_stat(os.stat(path)) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) def lstat(self, path): path = self._realpath(path) try: return paramiko.SFTPAttributes.from_stat(os.lstat(path)) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) def open(self, path, flags, attr): path = self._realpath(path) try: flags |= getattr(os, "O_BINARY", 0) if getattr(attr, "st_mode", None): fd = os.open(path, flags, attr.st_mode) else: # os.open() defaults to 0777 which is # an odd default mode for files fd = os.open(path, flags, 0o666) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) if (flags & os.O_CREAT) and (attr is not None): attr._flags &= ~attr.FLAG_PERMISSIONS paramiko.SFTPServer.set_file_attr(path, attr) if flags & os.O_WRONLY: fstr = "wb" elif flags & os.O_RDWR: fstr = "rb+" else: # O_RDONLY (== 0) fstr = "rb" try: f = os.fdopen(fd, fstr) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) fobj = StubSFTPHandle() fobj.filename = path fobj.readfile = f fobj.writefile = f return fobj def remove(self, path): path = self._realpath(path) try: os.remove(path) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK def rename(self, oldpath, newpath): oldpath = self._realpath(oldpath) newpath = self._realpath(newpath) try: os.rename(oldpath, newpath) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK def symlink(self, target_path, path): path = self._realpath(path) try: os.symlink(target_path, path) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK def readlink(self, path): path = self._realpath(path) try: target_path = os.readlink(path) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return target_path def mkdir(self, path, attr): path = self._realpath(path) try: # Using getattr() in case st_mode is None or 0 # both evaluate to False if getattr(attr, "st_mode", None): os.mkdir(path, attr.st_mode) else: os.mkdir(path) if attr is not None: attr._flags &= ~attr.FLAG_PERMISSIONS paramiko.SFTPServer.set_file_attr(path, attr) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK def rmdir(self, path): path = self._realpath(path) try: os.rmdir(path) except OSError as e: return paramiko.SFTPServer.convert_errno(e.errno) return paramiko.SFTP_OK # removed: chattr # (nothing in bzr's sftp transport uses those) # ------------- server test implementation -------------- STUB_SERVER_KEY = """\ -----BEGIN RSA PRIVATE KEY----- MIICWgIBAAKBgQDTj1bqB4WmayWNPB+8jVSYpZYk80Ujvj680pOTh2bORBjbIAyz oWGW+GUjzKxTiiPvVmxFgx5wdsFvF03v34lEVVhMpouqPAYQ15N37K/ir5XY+9m/ d8ufMCkjeXsQkKqFbAlQcnWMCRnOoPHS3I4vi6hmnDDeeYTSRvfLbW0fhwIBIwKB gBIiOqZYaoqbeD9OS9z2K9KR2atlTxGxOJPXiP4ESqP3NVScWNwyZ3NXHpyrJLa0 EbVtzsQhLn6rF+TzXnOlcipFvjsem3iYzCpuChfGQ6SovTcOjHV9z+hnpXvQ/fon soVRZY65wKnF7IAoUwTmJS9opqgrN6kRgCd3DASAMd1bAkEA96SBVWFt/fJBNJ9H tYnBKZGw0VeHOYmVYbvMSstssn8un+pQpUm9vlG/bp7Oxd/m+b9KWEh2xPfv6zqU avNwHwJBANqzGZa/EpzF4J8pGti7oIAPUIDGMtfIcmqNXVMckrmzQ2vTfqtkEZsA 4rE1IERRyiJQx6EJsz21wJmGV9WJQ5kCQQDwkS0uXqVdFzgHO6S++tjmjYcxwr3g H0CoFYSgbddOT6miqRskOQF3DZVkJT3kyuBgU2zKygz52ukQZMqxCb1fAkASvuTv qfpH87Qq5kQhNKdbbwbmd2NxlNabazPijWuphGTdW0VfJdWfklyS2Kr+iqrs/5wV HhathJt636Eg7oIjAkA8ht3MQ+XSl9yIJIS8gVpbPxSw5OMfw0PjVE7tBdQruiSc nvuQES5C9BMHjF39LZiGH1iLQy7FgdHyoP+eodI7 -----END RSA PRIVATE KEY----- """ class SocketDelay: """A socket decorator to make TCP appear slower. This changes recv, send, and sendall to add a fixed latency to each python call if a new roundtrip is detected. That is, when a recv is called and the flag new_roundtrip is set, latency is charged. Every send and send_all sets this flag. In addition every send, sendall and recv sleeps a bit per character send to simulate bandwidth. Not all methods are implemented, this is deliberate as this class is not a replacement for the builtin sockets layer. fileno is not implemented to prevent the proxy being bypassed. """ simulated_time = 0 _proxied_arguments = dict.fromkeys( [ "close", "getpeername", "getsockname", "getsockopt", "gettimeout", "setblocking", "setsockopt", "settimeout", "shutdown", ] ) def __init__(self, sock, latency, bandwidth=1.0, really_sleep=True): """:param bandwith: simulated bandwith (MegaBit) :param really_sleep: If set to false, the SocketDelay will just increase a counter, instead of calling time.sleep. This is useful for unittesting the SocketDelay. """ self.sock = sock self.latency = latency self.really_sleep = really_sleep self.time_per_byte = 1 / (bandwidth / 8.0 * 1024 * 1024) self.new_roundtrip = False def sleep(self, s): if self.really_sleep: time.sleep(s) else: SocketDelay.simulated_time += s def __getattr__(self, attr): if attr in SocketDelay._proxied_arguments: return getattr(self.sock, attr) raise AttributeError(f"'SocketDelay' object has no attribute {attr!r}") def dup(self): return SocketDelay( self.sock.dup(), self.latency, self.time_per_byte, self._sleep ) def recv(self, *args): data = self.sock.recv(*args) if data and self.new_roundtrip: self.new_roundtrip = False self.sleep(self.latency) self.sleep(len(data) * self.time_per_byte) return data def sendall(self, data, flags=0): if not self.new_roundtrip: self.new_roundtrip = True self.sleep(self.latency) self.sleep(len(data) * self.time_per_byte) return self.sock.sendall(data, flags) def send(self, data, flags=0): if not self.new_roundtrip: self.new_roundtrip = True self.sleep(self.latency) bytes_sent = self.sock.send(data, flags) self.sleep(bytes_sent * self.time_per_byte) return bytes_sent class TestingSFTPConnectionHandler(socketserver.BaseRequestHandler): def setup(self): self.wrap_for_latency() tcs = self.server.test_case_server ptrans = paramiko.Transport(self.request) self.paramiko_transport = ptrans # Set it to a channel under 'bzr' so that we get debug info ptrans.set_log_channel("brz.paramiko.transport") ptrans.add_server_key(tcs.get_host_key()) ptrans.set_subsystem_handler( "sftp", paramiko.SFTPServer, StubSFTPServer, root=tcs._root, home=tcs._server_homedir, ) server = tcs._server_interface(tcs) # This blocks until the key exchange has been done ptrans.start_server(None, server) def finish(self): # Wait for the conversation to finish, when the paramiko.Transport # thread finishes # TODO: Consider timing out after XX seconds rather than hanging. # Also we could check paramiko_transport.active and possibly # paramiko_transport.getException(). self.paramiko_transport.join() def wrap_for_latency(self): tcs = self.server.test_case_server if tcs.add_latency: # Give the socket (which the request really is) a latency adding # decorator. self.request = SocketDelay(self.request, tcs.add_latency) class TestingSFTPWithoutSSHConnectionHandler(TestingSFTPConnectionHandler): def setup(self): self.wrap_for_latency() # Re-import these as locals, so that they're still accessible during # interpreter shutdown (when all module globals get set to None, leading # to confusing errors like "'NoneType' object has no attribute 'error'". class FakeChannel: def __init__(self, sock): self._socket = sock def get_transport(self): return self def get_log_channel(self): return "brz.paramiko" def get_name(self): return "1" def get_hexdump(self): return False def close(self): # Close the underlying socket to ensure that any blocking recv() # calls will be interrupted and return, preventing hangs during # server shutdown. try: self._socket.close() except OSError: pass tcs = self.server.test_case_server fake_channel = FakeChannel(self.request) sftp_server = paramiko.SFTPServer( fake_channel, "sftp", StubServer(tcs), StubSFTPServer, root=tcs._root, home=tcs._server_homedir, ) self.sftp_server = sftp_server try: sftp_server.start_subsystem( "sftp", None, ssh.SocketAsChannelAdapter(self.request) ) except OSError as e: if (len(e.args) > 0) and (e.args[0] == errno.EPIPE): # it's okay for the client to disconnect abruptly # (bug in paramiko 1.6: it should absorb this exception) pass else: raise def finish(self): self.sftp_server.finish_subsystem() class TestingSFTPServer(test_server.TestingThreadingTCPServer): def __init__(self, server_address, request_handler_class, test_case_server): test_server.TestingThreadingTCPServer.__init__( self, server_address, request_handler_class ) self.test_case_server = test_case_server class SFTPServer(test_server.TestingTCPServerInAThread): """Common code for SFTP server facilities.""" def __init__(self, server_interface=StubServer): self.host = "127.0.0.1" self.port = 0 super().__init__( (self.host, self.port), TestingSFTPServer, TestingSFTPConnectionHandler ) self._original_vendor = None self._vendor = ParamikoVendor() self._server_interface = server_interface self._host_key = None self.logs = [] self.add_latency = 0 self._homedir = None self._server_homedir = None self._root = None def _get_sftp_url(self, path): """Calculate an sftp url to this server for path.""" return f"sftp://foo:bar@{self.host}:{self.port}/{path}" def log(self, message): """StubServer uses this to log when a new server is created.""" self.logs.append(message) def create_server(self): server = self.server_class( (self.host, self.port), self.request_handler_class, self ) return server def get_host_key(self): if self._host_key is None: key_file = osutils.pathjoin(self._homedir, "test_rsa.key") f = open(key_file, "w") try: f.write(STUB_SERVER_KEY) finally: f.close() self._host_key = paramiko.RSAKey.from_private_key_file(key_file) return self._host_key def start_server(self, backing_server=None): # XXX: TODO: make sftpserver back onto backing_server rather than local # disk. if not ( backing_server is None or isinstance(backing_server, test_server.LocalURLServer) ): raise AssertionError( "backing_server should not be {!r}, because this can only serve " "the local current working directory.".format(backing_server) ) self._original_vendor = ssh._ssh_vendor_manager._cached_ssh_vendor ssh._ssh_vendor_manager._cached_ssh_vendor = self._vendor self._homedir = os.getcwd() if sys.platform == "win32": # Normalize the path or it will be wrongly escaped self._homedir = osutils.normpath(self._homedir) else: self._homedir = self._homedir if self._server_homedir is None: self._server_homedir = self._homedir self._root = "/" if sys.platform == "win32": self._root = "" super().start_server() def stop_server(self): try: super().stop_server() finally: ssh._ssh_vendor_manager._cached_ssh_vendor = self._original_vendor def get_bogus_url(self): """See dromedary.Server.get_bogus_url.""" # this is chosen to try to prevent trouble with proxies, weird dns, etc # we bind a random socket, so that we get a guaranteed unused port # we just never listen on that port s = socket.socket() s.bind(("localhost", 0)) return "sftp://{}:{}/".format(*s.getsockname()) class SFTPFullAbsoluteServer(SFTPServer): """A test server for sftp transports, using absolute urls and ssh.""" def get_url(self): """See dromedary.Server.get_url.""" homedir = self._homedir if sys.platform != "win32": # Remove the initial '/' on all platforms but win32 homedir = homedir[1:] return self._get_sftp_url(urlutils.escape(homedir)) class SFTPServerWithoutSSH(SFTPServer): """An SFTP server that uses a simple TCP socket pair rather than SSH.""" def __init__(self): super().__init__() self._vendor = ssh.LoopbackVendor() self.request_handler_class = TestingSFTPWithoutSSHConnectionHandler def get_host_key(self): return None class SFTPAbsoluteServer(SFTPServerWithoutSSH): """A test server for sftp transports, using absolute urls.""" def get_url(self): """See dromedary.Server.get_url.""" homedir = self._homedir if sys.platform != "win32": # Remove the initial '/' on all platforms but win32 homedir = homedir[1:] return self._get_sftp_url(urlutils.escape(homedir)) class SFTPHomeDirServer(SFTPServerWithoutSSH): """A test server for sftp transports, using homedir relative urls.""" def get_url(self): """See dromedary.Server.get_url.""" return self._get_sftp_url("%7E/") class SFTPSiblingAbsoluteServer(SFTPAbsoluteServer): """A test server for sftp transports where only absolute paths will work. It does this by serving from a deeply-nested directory that doesn't exist. """ def create_server(self): # FIXME: Can't we do that in a cleaner way ? -- vila 20100623 server = super().create_server() server._server_homedir = "/dev/noone/runs/tests/here" return server dromedary-0.1.1/tests/test_cethread.py000064400000000000000000000122631046102023000161470ustar 00000000000000# Copyright (C) 2011, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import threading import unittest from dromedary import cethread class TestCatchingExceptionThread(unittest.TestCase): def test_start_and_join_smoke_test(self): def do_nothing(): pass tt = cethread.CatchingExceptionThread(target=do_nothing) tt.start() tt.join() def test_exception_is_re_raised(self): class MyException(Exception): pass def raise_my_exception(): raise MyException() tt = cethread.CatchingExceptionThread(target=raise_my_exception) tt.start() self.assertRaises(MyException, tt.join) def test_join_around_exception(self): resume = threading.Event() class MyException(Exception): pass def raise_my_exception(): # Wait for the test to tell us to resume resume.wait() # Now we can raise raise MyException() tt = cethread.CatchingExceptionThread(target=raise_my_exception) tt.start() tt.join(timeout=0) self.assertIs(None, tt.exception) resume.set() self.assertRaises(MyException, tt.join) def test_sync_event(self): control = threading.Event() in_thread = threading.Event() class MyException(Exception): pass def raise_my_exception(): # Wait for the test to tell us to resume control.wait() # Now we can raise raise MyException() tt = cethread.CatchingExceptionThread( target=raise_my_exception, sync_event=in_thread ) tt.start() tt.join(timeout=0) self.assertIs(None, tt.exception) self.assertIs(in_thread, tt.sync_event) control.set() self.assertRaises(MyException, tt.join) self.assertEqual(True, tt.sync_event.is_set()) def test_switch_and_set(self): """Caller can precisely control a thread.""" control1 = threading.Event() control2 = threading.Event() control3 = threading.Event() class TestThread(cethread.CatchingExceptionThread): def __init__(self): super().__init__(target=self.step_by_step) self.current_step = "starting" self.step1 = threading.Event() self.set_sync_event(self.step1) self.step2 = threading.Event() self.final = threading.Event() def step_by_step(self): control1.wait() self.current_step = "step1" self.switch_and_set(self.step2) control2.wait() self.current_step = "step2" self.switch_and_set(self.final) control3.wait() self.current_step = "done" tt = TestThread() tt.start() self.assertEqual("starting", tt.current_step) control1.set() tt.step1.wait() self.assertEqual("step1", tt.current_step) control2.set() tt.step2.wait() self.assertEqual("step2", tt.current_step) control3.set() # We don't wait on tt.final tt.join() self.assertEqual("done", tt.current_step) def test_exception_while_switch_and_set(self): control1 = threading.Event() class MyException(Exception): pass class TestThread(cethread.CatchingExceptionThread): def __init__(self, *args, **kwargs): self.step1 = threading.Event() self.step2 = threading.Event() super().__init__(target=self.step_by_step, sync_event=self.step1) self.current_step = "starting" self.set_sync_event(self.step1) def step_by_step(self): control1.wait() self.current_step = "step1" self.switch_and_set(self.step2) def set_sync_event(self, event): # We force an exception while trying to set step2 if event is self.step2: raise MyException() super().set_sync_event(event) tt = TestThread() tt.start() self.assertEqual("starting", tt.current_step) control1.set() # We now wait on step1 which will be set when catching the exception tt.step1.wait() self.assertRaises(MyException, tt.pending_exception) self.assertIs(tt.step1, tt.sync_event) self.assertTrue(tt.step1.is_set()) dromedary-0.1.1/tests/test_errors.py000064400000000000000000000046541046102023000157110ustar 00000000000000# Copyright (C) 2006-2012, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Tests for dromedary error classes.""" import unittest from dromedary.errors import SocketConnectionError class TestSocketConnectionError(unittest.TestCase): def assertSocketConnectionError(self, expected, *args, **kwargs): e = SocketConnectionError(*args, **kwargs) self.assertEqual(expected, str(e)) def test_default(self): self.assertSocketConnectionError("Failed to connect to ahost", "ahost") def test_port_none(self): self.assertSocketConnectionError( "Failed to connect to ahost", "ahost", port=None ) def test_port_supplied(self): self.assertSocketConnectionError( "Failed to connect to ahost:22", "ahost", port=22 ) def test_with_orig_error_and_port(self): self.assertSocketConnectionError( "Failed to connect to ahost:22; bogus error", "ahost", port=22, orig_error="bogus error", ) def test_with_orig_error_no_port(self): self.assertSocketConnectionError( "Failed to connect to ahost; bogus error", "ahost", orig_error="bogus error", ) def test_orig_error_exception_object(self): orig_error = ValueError("bad value") self.assertSocketConnectionError( f"Failed to connect to ahost; {orig_error!s}", host="ahost", orig_error=orig_error, ) def test_custom_msg(self): self.assertSocketConnectionError( "Unable to connect to ssh host ahost:444; my_error", host="ahost", port=444, msg="Unable to connect to ssh host", orig_error="my_error", ) dromedary-0.1.1/tests/test_gio_transport.py000064400000000000000000000056221046102023000172630ustar 00000000000000# Copyright (C) 2025 Breezy Developers # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. """Tests for the GIO transport. These exercise the `gio+file://` backend, which is the only gio backend that works without a real gvfs mount. The whole module is skipped when dromedary was built without the `gio` Cargo feature. """ import tempfile import unittest from dromedary import urlutils from dromedary.errors import DependencyNotPresent try: from dromedary.gio_transport import GioTransport except DependencyNotPresent: GioTransport = None @unittest.skipIf(GioTransport is None, "dromedary built without the gio feature") class GioTransportTests(unittest.TestCase): def setUp(self): if GioTransport is None: self.skipTest("dromedary built without the gio feature") self._dir = tempfile.TemporaryDirectory() self.addCleanup(self._dir.cleanup) self.base = "gio+" + urlutils.local_path_to_url(self._dir.name) + "/" self.t = GioTransport(self.base) def test_external_url_round_trips(self): self.assertEqual(self.base, self.t.external_url()) def test_put_get_has(self): self.assertFalse(self.t.has("hello")) self.t.put_bytes("hello", b"world") self.assertTrue(self.t.has("hello")) self.assertEqual(b"world", self.t.get_bytes("hello")) def test_mkdir_stat_list(self): self.t.mkdir("d") self.t.put_bytes("d/a", b"1") self.t.put_bytes("d/b", b"22") self.assertEqual(["a", "b"], sorted(self.t.list_dir("d"))) st = self.t.stat("d/a") self.assertEqual(1, st.st_size) def test_rename_and_delete(self): self.t.put_bytes("a", b"hi") self.t.rename("a", "b") self.assertFalse(self.t.has("a")) self.assertEqual(b"hi", self.t.get_bytes("b")) self.t.delete("b") self.assertFalse(self.t.has("b")) def test_append_extends_file(self): self.t.put_bytes("f", b"abc") from io import BytesIO offset = self.t.append_file("f", BytesIO(b"DEF")) self.assertEqual(3, offset) self.assertEqual(b"abcDEF", self.t.get_bytes("f")) def test_clone_descends(self): self.t.mkdir("sub") self.t.put_bytes("sub/inside", b"x") sub = self.t.clone("sub") self.assertEqual(b"x", sub.get_bytes("inside")) def test_missing_file_raises(self): from dromedary.errors import NoSuchFile self.assertRaises(NoSuchFile, self.t.get_bytes, "nope") dromedary-0.1.1/tests/test_server.py000064400000000000000000000416231046102023000157000ustar 00000000000000# Copyright (C) 2010, 2011 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Test server implementations for transport decorators and TCP testing.""" import errno import socket import socketserver import sys import threading import dromedary from dromedary import chroot, pathfilter, urlutils from dromedary.cethread import CatchingExceptionThread def connect_socket(address): """Connect to the given address, trying all results from getaddrinfo.""" err = OSError("getaddrinfo returns an empty list") host, port = address for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, _canonname, sa = res sock = None try: sock = socket.socket(af, socktype, proto) sock.connect(sa) return sock except OSError as e: err = e if sock is not None: sock.close() raise err # Set by test frameworks to enable debug output for thread operations. debug_threads_hook = None def debug_threads(): """Return True if thread debugging is enabled.""" if debug_threads_hook is not None: return debug_threads_hook() return False class TestServer(dromedary.Server): """A Transport Server dedicated to tests. The TestServer interface provides a server for a given transport. We use these servers as loopback testing tools. For any given transport the Servers it provides must either allow writing, or serve the contents of osutils.getcwd() at the time start_server is called. Note that these are real servers - they must implement all the things that we want bzr transports to take advantage of. """ def get_url(self): """Return a url for this server.""" raise NotImplementedError def get_bogus_url(self): """Return a url for this protocol, that will fail to connect.""" raise NotImplementedError class LocalURLServer(TestServer): """A pretend server for local transports, using file:// urls.""" def start_server(self): pass def get_url(self): return urlutils.local_path_to_url("") class DecoratorServer(TestServer): """Server for the TransportDecorator for testing with.""" def start_server(self, server=None): if server is not None: self._made_server = False self._server = server else: self._made_server = True self._server = LocalURLServer() self._server.start_server() def stop_server(self): if self._made_server: self._server.stop_server() def get_decorator_class(self): raise NotImplementedError(self.get_decorator_class) def get_url_prefix(self): return self.get_decorator_class()._get_url_prefix() def get_bogus_url(self): return self.get_url_prefix() + self._server.get_bogus_url() def get_url(self): return self.get_url_prefix() + self._server.get_url() class BrokenRenameServer(DecoratorServer): """Server for the BrokenRenameTransportDecorator for testing with.""" def get_decorator_class(self): from dromedary import brokenrename return brokenrename.BrokenRenameTransportDecorator class FakeNFSServer(DecoratorServer): """Server for the FakeNFSTransportDecorator for testing with.""" def get_decorator_class(self): from dromedary import fakenfs return fakenfs.FakeNFSTransportDecorator class FakeVFATServer(DecoratorServer): """A server that suggests connections through FakeVFATTransportDecorator.""" def get_decorator_class(self): from dromedary import fakevfat return fakevfat.FakeVFATTransportDecorator class LogDecoratorServer(DecoratorServer): """Server for testing.""" def get_decorator_class(self): from dromedary import log return log.TransportLogDecorator class ReadonlyServer(DecoratorServer): """Server for the ReadonlyTransportDecorator for testing with.""" def get_decorator_class(self): from dromedary import readonly return readonly.ReadonlyTransportDecorator class TraceServer(DecoratorServer): """Server for the TransportTraceDecorator for testing with.""" def get_decorator_class(self): from dromedary import trace return trace.TransportTraceDecorator class UnlistableServer(DecoratorServer): """Server for the UnlistableTransportDecorator for testing with.""" def get_decorator_class(self): from dromedary import unlistable return unlistable.UnlistableTransportDecorator class TestingPathFilteringServer(pathfilter.PathFilteringServer): def __init__(self): """TestingPathFilteringServer is not usable until start_server is called. """ def start_server(self, backing_server=None): """Setup the Chroot on backing_server.""" if backing_server is not None: self.backing_transport = dromedary.get_transport_from_url( backing_server.get_url() ) else: self.backing_transport = dromedary.get_transport_from_path(".") self.backing_transport.clone("added-by-filter").ensure_base() self.filter_func = lambda x: "added-by-filter/" + x super().start_server() def get_bogus_url(self): raise NotImplementedError class TestingChrootServer(chroot.ChrootServer): def __init__(self): """TestingChrootServer is not usable until start_server is called.""" super().__init__(None) def start_server(self, backing_server=None): """Setup the Chroot on backing_server.""" if backing_server is not None: self.backing_transport = dromedary.get_transport_from_url( backing_server.get_url() ) else: self.backing_transport = dromedary.get_transport_from_path(".") super().start_server() def get_bogus_url(self): raise NotImplementedError class TestThread(CatchingExceptionThread): def join(self, timeout=5): """Overrides to use a default timeout. The default timeout is set to 5 and should expire only when a thread serving a client connection is hung. """ super().join(timeout) if timeout and self.is_alive(): # The timeout expired without joining the thread, the thread is # therefore stucked and that's a failure as far as the test is # concerned. We used to hang here. # FIXME: we need to kill the thread, but as far as the test is # concerned, raising an assertion is too strong. On most of the # platforms, this doesn't occur, so just mentioning the problem is # enough for now -- vila 2010824 sys.stderr.write(f"thread {self.name} hung\n") class TestingTCPServerMixin: """Mixin to support running socketserver.TCPServer in a thread. Tests are connecting from the main thread, the server has to be run in a separate thread. """ def __init__(self): self.started = threading.Event() self.serving = None self.stopped = threading.Event() self.clients = [] self.ignored_exceptions = None def server_bind(self): self.socket.bind(self.server_address) self.server_address = self.socket.getsockname() def serve(self): self.serving = True self.started.set() try: while self.serving: self.handle_request() self.server_close() finally: self.stopped.set() def handle_request(self): """Handle one request. The python version swallows some socket exceptions and we don't use timeout, so we override it to better control the server behavior. """ request, client_address = self.get_request() if self.verify_request(request, client_address): try: self.process_request(request, client_address) except BaseException: self.handle_error(request, client_address) else: self.close_request(request) def get_request(self): return self.socket.accept() def verify_request(self, request, client_address): """Verify the request. Return True if we should proceed with this request, False if we should not even touch a single byte in the socket ! This is useful when we stop the server with a dummy last connection. """ return self.serving def handle_error(self, request, client_address): # Stop serving and re-raise the last exception seen self.serving = False # We call close_request manually, because we are going to raise an # exception. The socketserver implementation calls: # handle_error(...) # close_request(...) # But because we raise the exception, close_request will never be # triggered. This helps client not block waiting for a response when # the server gets an exception. self.close_request(request) raise def ignored_exceptions_during_shutdown(self, e): if sys.platform == "win32": accepted_errnos = [ errno.EBADF, errno.EPIPE, errno.WSAEBADF, errno.WSAENOTSOCK, errno.WSAECONNRESET, errno.WSAENOTCONN, errno.WSAESHUTDOWN, ] else: accepted_errnos = [ errno.EBADF, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE, ] return bool(isinstance(e, socket.error) and e.errno in accepted_errnos) def stop_client_connections(self): while self.clients: c = self.clients.pop() self.shutdown_client(c) def shutdown_socket(self, sock): """Properly shutdown a socket. This should be called only when no other thread is trying to use the socket. """ try: sock.shutdown(socket.SHUT_RDWR) sock.close() except Exception as e: if self.ignored_exceptions(e): pass else: raise def set_ignored_exceptions(self, thread, ignored_exceptions): self.ignored_exceptions = ignored_exceptions thread.set_ignored_exceptions(self.ignored_exceptions) def _pending_exception(self, thread): """Raise server uncaught exception. Daughter classes can override this if they use daughter threads. """ thread.pending_exception() class TestingTCPServer(TestingTCPServerMixin, socketserver.TCPServer): def __init__(self, server_address, request_handler_class): TestingTCPServerMixin.__init__(self) socketserver.TCPServer.__init__(self, server_address, request_handler_class) def get_request(self): """Get the request and client address from the socket.""" sock, addr = TestingTCPServerMixin.get_request(self) self.clients.append((sock, addr)) return sock, addr def shutdown_client(self, client): sock, _addr = client self.shutdown_socket(sock) class TestingThreadingTCPServer(TestingTCPServerMixin, socketserver.ThreadingTCPServer): def __init__(self, server_address, request_handler_class): TestingTCPServerMixin.__init__(self) socketserver.ThreadingTCPServer.__init__( self, server_address, request_handler_class ) def get_request(self): """Get the request and client address from the socket.""" sock, addr = TestingTCPServerMixin.get_request(self) self.clients.append((sock, addr, None)) return sock, addr def process_request_thread( self, started, detached, stopped, request, client_address ): started.set() detached.wait() socketserver.ThreadingTCPServer.process_request_thread( self, request, client_address ) self.close_request(request) stopped.set() def process_request(self, request, client_address): """Start a new thread to process the request.""" started = threading.Event() detached = threading.Event() stopped = threading.Event() t = TestThread( sync_event=stopped, name=f"{client_address} -> {self.server_address}", target=self.process_request_thread, args=(started, detached, stopped, request, client_address), ) self.clients.pop() self.clients.append((request, client_address, t)) t.set_ignored_exceptions(self.ignored_exceptions) t.start() started.wait() t.pending_exception() if debug_threads(): sys.stderr.write(f"Client thread {t.name} started\n") detached.set() def shutdown_client(self, client): sock, _addr, connection_thread = client self.shutdown_socket(sock) if connection_thread is not None: if debug_threads(): sys.stderr.write( f"Client thread {connection_thread.name} will be joined\n" ) connection_thread.join() def set_ignored_exceptions(self, thread, ignored_exceptions): TestingTCPServerMixin.set_ignored_exceptions(self, thread, ignored_exceptions) for _sock, _addr, connection_thread in self.clients: if connection_thread is not None: connection_thread.set_ignored_exceptions(self.ignored_exceptions) def _pending_exception(self, thread): for _sock, _addr, connection_thread in self.clients: if connection_thread is not None: connection_thread.pending_exception() TestingTCPServerMixin._pending_exception(self, thread) class TestingTCPServerInAThread(dromedary.Server): """A server in a thread that re-raise thread exceptions.""" def __init__(self, server_address, server_class, request_handler_class): self.server_class = server_class self.request_handler_class = request_handler_class self.host, self.port = server_address self.server = None self._server_thread = None def __repr__(self): return f"{self.__class__.__name__}({self.host}:{self.port})" def create_server(self): return self.server_class((self.host, self.port), self.request_handler_class) def start_server(self): self.server = self.create_server() self._server_thread = TestThread( sync_event=self.server.started, target=self.run_server ) self._server_thread.start() self.server.started.wait() self.host, self.port = self.server.server_address self._server_thread.name = self.server.server_address if debug_threads(): sys.stderr.write(f"Server thread {self._server_thread.name} started\n") self._server_thread.pending_exception() self._server_thread.set_sync_event(self.server.stopped) def run_server(self): self.server.serve() def stop_server(self): if self.server is None: return try: self.set_ignored_exceptions(self.server.ignored_exceptions_during_shutdown) self.server.serving = False if debug_threads(): sys.stderr.write( f"Server thread {self._server_thread.name} will be joined\n" ) last_conn = None try: last_conn = connect_socket((self.host, self.port)) except OSError: pass self.server.stop_client_connections() self.server.stopped.wait() if last_conn is not None: last_conn.close() try: self._server_thread.join() except Exception as e: if self.server.ignored_exceptions(e): pass else: raise finally: self.server = None def set_ignored_exceptions(self, ignored_exceptions): """Install an exception handler for the server.""" self.server.set_ignored_exceptions(self._server_thread, ignored_exceptions) def pending_exception(self): """Raise uncaught exception in the server.""" self.server._pending_exception(self._server_thread) dromedary-0.1.1/tests/test_test_server.py000064400000000000000000000202661046102023000167370ustar 00000000000000# Copyright (C) 2010, 2011, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import errno import socket import socketserver import threading import unittest from dromedary.tests import test_server def portable_socket_pair(): """Return a pair of TCP sockets connected to each other. Unlike socket.socketpair, this should work on Windows. """ listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) listen_sock.bind(("127.0.0.1", 0)) listen_sock.listen(1) client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client_sock.connect(listen_sock.getsockname()) server_sock, _addr = listen_sock.accept() listen_sock.close() return server_sock, client_sock class TCPClient: def __init__(self): self.sock = None def connect(self, addr): if self.sock is not None: raise AssertionError(f"Already connected to {self.sock.getsockname()!r}") self.sock = test_server.connect_socket(addr) def disconnect(self): if self.sock is not None: try: self.sock.shutdown(socket.SHUT_RDWR) self.sock.close() except OSError as e: if e.errno in (errno.EBADF, errno.ENOTCONN, errno.ECONNRESET): pass else: raise self.sock = None def write(self, s): return self.sock.sendall(s) def read(self, bufsize=4096): try: return self.sock.recv(bufsize) except OSError as e: if e.errno == errno.ECONNRESET: return b"" raise class TCPConnectionHandler(socketserver.BaseRequestHandler): def handle(self): self.done = False self.handle_connection() while not self.done: self.handle_connection() def readline(self): req = self.request.recv(4096) if not req or (req.endswith(b"\n") and req.count(b"\n") == 1): return req raise ValueError(f"[{req!r}] not a simple line") def handle_connection(self): req = self.readline() if not req: self.done = True elif req == b"ping\n": self.request.sendall(b"pong\n") else: raise ValueError(f"[{req}] not understood") class TestTCPServerInAThreadBase: """Mixin with test methods for TCP server implementations.""" server_class = None def get_server(self, server_class=None, connection_handler_class=None): if server_class is not None: self.server_class = server_class if connection_handler_class is None: connection_handler_class = TCPConnectionHandler server = test_server.TestingTCPServerInAThread( ("localhost", 0), self.server_class, connection_handler_class ) server.start_server() self.addCleanup(server.stop_server) return server def get_client(self): client = TCPClient() self.addCleanup(client.disconnect) return client def get_server_connection(self, server, conn_rank): return server.server.clients[conn_rank] def assertClientAddr(self, client, server, conn_rank): conn = self.get_server_connection(server, conn_rank) self.assertEqual(client.sock.getsockname(), conn[1]) def test_start_stop(self): server = self.get_server() client = self.get_client() server.stop_server() # since the server doesn't accept connections anymore attempting to # connect should fail client = self.get_client() self.assertRaises(socket.error, client.connect, (server.host, server.port)) def test_client_talks_server_respond(self): server = self.get_server() client = self.get_client() client.connect((server.host, server.port)) self.assertIs(None, client.write(b"ping\n")) resp = client.read() self.assertClientAddr(client, server, 0) self.assertEqual(b"pong\n", resp) def test_server_fails_to_start(self): class CantStart(Exception): pass class CantStartServer(test_server.TestingTCPServer): def server_bind(self): raise CantStart() # The exception is raised in the main thread self.assertRaises(CantStart, self.get_server, server_class=CantStartServer) def test_server_fails_while_serving_or_stopping(self): class CantConnect(Exception): pass class FailingConnectionHandler(TCPConnectionHandler): def handle(self): raise CantConnect() server = self.get_server(connection_handler_class=FailingConnectionHandler) client = self.get_client() client.connect((server.host, server.port)) client.write(b"ping\n") try: self.assertEqual(b"", client.read()) except OSError as e: WSAECONNRESET = 10054 if e.errno in (WSAECONNRESET,): pass self.assertRaises(CantConnect, server.stop_server) def test_server_crash_while_responding(self): caught = threading.Event() caught.clear() self.connection_thread = None class FailToRespond(Exception): pass class FailingDuringResponseHandler(TCPConnectionHandler): def handle_connection(request): # noqa: N805 request.readline() self.connection_thread = threading.current_thread() self.connection_thread.set_sync_event(caught) raise FailToRespond() server = self.get_server(connection_handler_class=FailingDuringResponseHandler) client = self.get_client() client.connect((server.host, server.port)) client.write(b"ping\n") caught.wait() self.assertEqual(b"", client.read()) self.assertRaises(FailToRespond, self.connection_thread.pending_exception) def test_exception_swallowed_while_serving(self): caught = threading.Event() caught.clear() self.connection_thread = None class CantServe(Exception): pass class FailingWhileServingConnectionHandler(TCPConnectionHandler): def handle(request): # noqa: N805 self.connection_thread = threading.current_thread() self.connection_thread.set_sync_event(caught) raise CantServe() server = self.get_server( connection_handler_class=FailingWhileServingConnectionHandler ) self.assertEqual(True, server.server.serving) server.set_ignored_exceptions(CantServe) client = self.get_client() client.connect((server.host, server.port)) caught.wait() self.assertEqual(b"", client.read()) self.assertIs(None, self.connection_thread.pending_exception()) self.assertIs(None, server.pending_exception()) def test_handle_request_closes_if_it_doesnt_process(self): server = self.get_server() client = self.get_client() server.server.serving = False try: client.connect((server.host, server.port)) self.assertEqual(b"", client.read()) except OSError as e: if e.errno != errno.ECONNRESET: raise class TestTCPServerInAThread_TestingTCPServer( TestTCPServerInAThreadBase, unittest.TestCase ): server_class = test_server.TestingTCPServer class TestTCPServerInAThread_TestingThreadingTCPServer( TestTCPServerInAThreadBase, unittest.TestCase ): server_class = test_server.TestingThreadingTCPServer dromedary-0.1.1/tests/test_transport.py000064400000000000000000001320621046102023000164240ustar 00000000000000# Copyright (C) 2005-2011, 2015, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import errno import os import subprocess import sys import threading from io import BytesIO import dromedary as transport from dromedary import ( chroot, errors, fakenfs, local, memory, osutils, pathfilter, readonly, tests, urlutils, ) from dromedary import tests as features from dromedary.errors import FileExists, NoSuchFile, UnsupportedProtocol from dromedary.local import file_kind from dromedary.tests import test_server # TODO: Should possibly split transport-specific tests into their own files. class TestTransport(tests.TestCase): """Test the non transport-concrete class functionality.""" def test__get_set_protocol_handlers(self): handlers = transport._get_protocol_handlers() self.assertNotEqual([], handlers.keys()) transport._clear_protocol_handlers() self.addCleanup(transport._set_protocol_handlers, handlers) self.assertEqual([], transport._get_protocol_handlers().keys()) def test_get_transport_modules(self): handlers = transport._get_protocol_handlers() self.addCleanup(transport._set_protocol_handlers, handlers) # don't pollute the current handlers transport._clear_protocol_handlers() class SampleHandler: """I exist, isnt that enough?""" transport._clear_protocol_handlers() transport.register_transport_proto("foo") transport.register_lazy_transport( "foo", "dromedary.tests.test_transport", "TestTransport.SampleHandler" ) transport.register_transport_proto("bar") transport.register_lazy_transport( "bar", "dromedary.tests.test_transport", "TestTransport.SampleHandler" ) self.assertCountEqual( [ SampleHandler.__module__, "dromedary.chroot", "dromedary.pathfilter", ], transport._get_transport_modules(), ) def test_transport_dependency(self): """Transport with missing dependency causes no error.""" saved_handlers = transport._get_protocol_handlers() self.addCleanup(transport._set_protocol_handlers, saved_handlers) # don't pollute the current handlers transport._clear_protocol_handlers() transport.register_transport_proto("foo") transport.register_lazy_transport( "foo", "dromedary.tests.test_transport", "BadTransportHandler" ) try: transport.get_transport_from_url("foo://fooserver/foo") except UnsupportedProtocol as e: self.assertEqual( "Unsupported protocol" ' for url "foo://fooserver/foo":' ' Unable to import library "some_lib":' " testing missing dependency", str(e), ) else: self.fail("Did not raise UnsupportedProtocol") def test_transport_fallback(self): """Transport with missing dependency causes no error.""" saved_handlers = transport._get_protocol_handlers() self.addCleanup(transport._set_protocol_handlers, saved_handlers) transport._clear_protocol_handlers() transport.register_transport_proto("foo") transport.register_lazy_transport( "foo", "dromedary.tests.test_transport", "BackupTransportHandler" ) transport.register_lazy_transport( "foo", "dromedary.tests.test_transport", "BadTransportHandler" ) t = transport.get_transport_from_url("foo://fooserver/foo") self.assertIsInstance(t, BackupTransportHandler) def test_ssh_hints(self): """Transport ssh:// should raise UnsupportedProtocol.""" self.assertRaises( UnsupportedProtocol, transport.get_transport_from_url, "ssh://fooserver/foo", ) def test_LateReadError(self): """The LateReadError helper should raise on read().""" a_file = transport.LateReadError("a path") try: a_file.read() except errors.ReadError as error: self.assertEqual("a path", error.path) self.assertRaises(errors.ReadError, a_file.read, 40) a_file.close() def test_local_abspath_non_local_transport(self): # the base implementation should throw t = memory.MemoryTransport() e = self.assertRaises(errors.NotLocalUrl, t.local_abspath, "t") self.assertEqual("memory:///t is not a local path.", str(e)) class TestCoalesceOffsets(tests.TestCase): def check(self, expected, offsets, limit=0, max_size=0, fudge=0): coalesce = transport.Transport._coalesce_offsets exp = [transport._CoalescedOffset(*x) for x in expected] out = list( coalesce(offsets, limit=limit, fudge_factor=fudge, max_size=max_size) ) self.assertEqual(exp, out) def test_coalesce_empty(self): self.check([], []) def test_coalesce_simple(self): self.check([(0, 10, [(0, 10)])], [(0, 10)]) def test_coalesce_unrelated(self): self.check( [ (0, 10, [(0, 10)]), (20, 10, [(0, 10)]), ], [(0, 10), (20, 10)], ) def test_coalesce_unsorted(self): self.check([(0, 10, [(0, 10)]), (20, 10, [(0, 10)])], [(20, 10), (0, 10)]) def test_coalesce_nearby(self): self.check([(0, 20, [(0, 10), (10, 10)])], [(0, 10), (10, 10)]) def test_coalesce_overlapped(self): self.assertRaises( ValueError, self.check, [(0, 15, [(0, 10), (5, 10)])], [(0, 10), (5, 10)] ) def test_coalesce_limit(self): self.check( [ (10, 50, [(0, 10), (10, 10), (20, 10), (30, 10), (40, 10)]), (60, 50, [(0, 10), (10, 10), (20, 10), (30, 10), (40, 10)]), ], [ (10, 10), (20, 10), (30, 10), (40, 10), (50, 10), (60, 10), (70, 10), (80, 10), (90, 10), (100, 10), ], limit=5, ) def test_coalesce_no_limit(self): self.check( [ ( 10, 100, [ (0, 10), (10, 10), (20, 10), (30, 10), (40, 10), (50, 10), (60, 10), (70, 10), (80, 10), (90, 10), ], ), ], [ (10, 10), (20, 10), (30, 10), (40, 10), (50, 10), (60, 10), (70, 10), (80, 10), (90, 10), (100, 10), ], ) def test_coalesce_fudge(self): self.check( [ (10, 30, [(0, 10), (20, 10)]), (100, 10, [(0, 10)]), ], [(10, 10), (30, 10), (100, 10)], fudge=10, ) def test_coalesce_max_size(self): self.check( [ (10, 20, [(0, 10), (10, 10)]), (30, 50, [(0, 50)]), # If one range is above max_size, it gets its own coalesced # offset (100, 80, [(0, 80)]), ], [(10, 10), (20, 10), (30, 50), (100, 80)], max_size=50, ) def test_coalesce_no_max_size(self): self.check( [(10, 170, [(0, 10), (10, 10), (20, 50), (70, 100)])], [(10, 10), (20, 10), (30, 50), (80, 100)], ) def test_coalesce_default_limit(self): # By default we use a 100MB max size. ten_mb = 10 * 1024 * 1024 self.check( [ (0, 10 * ten_mb, [(i * ten_mb, ten_mb) for i in range(10)]), (10 * ten_mb, ten_mb, [(0, ten_mb)]), ], [(i * ten_mb, ten_mb) for i in range(11)], ) self.check( [(0, 11 * ten_mb, [(i * ten_mb, ten_mb) for i in range(11)])], [(i * ten_mb, ten_mb) for i in range(11)], max_size=1 * 1024 * 1024 * 1024, ) class TestMemoryServer(tests.TestCase): def test_create_server(self): server = memory.MemoryServer() server.start_server() url = server.get_url() self.assertIn(url, transport.transport_list_registry) t = transport.get_transport_from_url(url) del t server.stop_server() self.assertNotIn(url, transport.transport_list_registry) self.assertRaises(UnsupportedProtocol, transport.get_transport_from_url, url) class TestMemoryTransport(tests.TestCase): def test_get_transport(self): memory.MemoryTransport() def test_clone(self): t = memory.MemoryTransport() self.assertIsInstance(t, memory.MemoryTransport) self.assertEqual("memory:///", t.clone("/").base) def test_abspath(self): t = memory.MemoryTransport() self.assertEqual("memory:///relpath", t.abspath("relpath")) def test_abspath_of_root(self): t = memory.MemoryTransport() self.assertEqual("memory:///", t.base) self.assertEqual("memory:///", t.abspath("/")) def test_abspath_of_relpath_starting_at_root(self): t = memory.MemoryTransport() self.assertEqual("memory:///foo", t.abspath("/foo")) def test_append_and_get(self): t = memory.MemoryTransport() t.append_bytes("path", b"content") self.assertEqual(t.get("path").read(), b"content") t.append_file("path", BytesIO(b"content")) with t.get("path") as f: self.assertEqual(f.read(), b"contentcontent") def test_put_and_get(self): t = memory.MemoryTransport() t.put_file("path", BytesIO(b"content")) self.assertEqual(t.get("path").read(), b"content") t.put_bytes("path", b"content") self.assertEqual(t.get("path").read(), b"content") def test_append_without_dir_fails(self): t = memory.MemoryTransport() self.assertRaises(NoSuchFile, t.append_bytes, "dir/path", b"content") def test_put_without_dir_fails(self): t = memory.MemoryTransport() self.assertRaises(NoSuchFile, t.put_file, "dir/path", BytesIO(b"content")) def test_get_missing(self): transport = memory.MemoryTransport() self.assertRaises(NoSuchFile, transport.get, "foo") def test_has_missing(self): t = memory.MemoryTransport() self.assertEqual(False, t.has("foo")) def test_has_present(self): t = memory.MemoryTransport() t.append_bytes("foo", b"content") self.assertEqual(True, t.has("foo")) def test_list_dir(self): t = memory.MemoryTransport() t.put_bytes("foo", b"content") t.mkdir("dir") t.put_bytes("dir/subfoo", b"content") t.put_bytes("dirlike", b"content") self.assertEqual(["dir", "dirlike", "foo"], sorted(t.list_dir("."))) self.assertEqual(["subfoo"], sorted(t.list_dir("dir"))) def test_mkdir(self): t = memory.MemoryTransport() t.mkdir("dir") t.append_bytes("dir/path", b"content") with t.get("dir/path") as f: self.assertEqual(f.read(), b"content") def test_mkdir_missing_parent(self): t = memory.MemoryTransport() self.assertRaises(NoSuchFile, t.mkdir, "dir/dir") def test_mkdir_twice(self): t = memory.MemoryTransport() t.mkdir("dir") self.assertRaises(FileExists, t.mkdir, "dir") def test_parameters(self): t = memory.MemoryTransport() self.assertEqual(True, t.listable()) self.assertEqual(False, t.is_readonly()) def test_iter_files_recursive(self): t = memory.MemoryTransport() t.mkdir("dir") t.put_bytes("dir/foo", b"content") t.put_bytes("dir/bar", b"content") t.put_bytes("bar", b"content") paths = set(t.iter_files_recursive()) self.assertEqual({"dir/foo", "dir/bar", "bar"}, paths) def test_stat(self): t = memory.MemoryTransport() t.put_bytes("foo", b"content") t.put_bytes("bar", b"phowar") self.assertEqual(7, t.stat("foo").st_size) self.assertEqual(6, t.stat("bar").st_size) class ChrootDecoratorTransportTest(tests.TestCase): """Chroot decoration specific tests.""" def test_abspath(self): # The abspath is always relative to the chroot_url. server = chroot.ChrootServer( transport.get_transport_from_url("memory:///foo/bar/") ) self.start_server(server) t = transport.get_transport_from_url(server.get_url()) self.assertEqual(server.get_url(), t.abspath("/")) subdir_t = t.clone("subdir") self.assertEqual(server.get_url(), subdir_t.abspath("/")) def test_clone(self): server = chroot.ChrootServer( transport.get_transport_from_url("memory:///foo/bar/") ) self.start_server(server) t = transport.get_transport_from_url(server.get_url()) # relpath from root and root path are the same relpath_cloned = t.clone("foo") abspath_cloned = t.clone("/foo") self.assertEqual(server, relpath_cloned.server) self.assertEqual(server, abspath_cloned.server) def test_chroot_url_preserves_chroot(self): """Calling get_transport on a chroot transport's base should produce a transport with exactly the same behaviour as the original chroot transport. This is so that it is not possible to escape a chroot by doing:: url = chroot_transport.base parent_url = urlutils.join(url, '..') new_t = transport.get_transport_from_url(parent_url) """ server = chroot.ChrootServer( transport.get_transport_from_url("memory:///path/subpath") ) self.start_server(server) t = transport.get_transport_from_url(server.get_url()) new_t = transport.get_transport_from_url(t.base) self.assertEqual(t.server, new_t.server) self.assertEqual(t.base, new_t.base) def test_urljoin_preserves_chroot(self): """Using urlutils.join(url, '..') on a chroot URL should not produce a URL that escapes the intended chroot. This is so that it is not possible to escape a chroot by doing:: url = chroot_transport.base parent_url = urlutils.join(url, '..') new_t = transport.get_transport_from_url(parent_url) """ server = chroot.ChrootServer( transport.get_transport_from_url("memory:///path/") ) self.start_server(server) t = transport.get_transport_from_url(server.get_url()) self.assertRaises(urlutils.InvalidURLJoin, urlutils.join, t.base, "..") class TestChrootServer(tests.TestCase): def test_construct(self): backing_transport = memory.MemoryTransport() server = chroot.ChrootServer(backing_transport) self.assertEqual(backing_transport, server.backing_transport) def test_setUp(self): backing_transport = memory.MemoryTransport() server = chroot.ChrootServer(backing_transport) server.start_server() self.addCleanup(server.stop_server) self.assertIn(server.scheme, transport._get_protocol_handlers().keys()) def test_stop_server(self): backing_transport = memory.MemoryTransport() server = chroot.ChrootServer(backing_transport) server.start_server() server.stop_server() self.assertNotIn(server.scheme, transport._get_protocol_handlers().keys()) def test_get_url(self): backing_transport = memory.MemoryTransport() server = chroot.ChrootServer(backing_transport) server.start_server() self.addCleanup(server.stop_server) self.assertEqual("chroot-%d:///" % id(server), server.get_url()) class TestHooks(tests.TestCase): """Basic tests for transport hooks.""" def _get_connected_transport(self): return transport.ConnectedTransport("bogus:nowhere") def test_transporthooks_initialisation(self): """Check all expected transport hook points are set up.""" hookpoint = transport.TransportHooks() self.assertIn("post_connect", hookpoint, f"post_connect not in {hookpoint}") def test_post_connect(self): """Ensure the post_connect hook is called when _set_transport is.""" calls = [] transport.Transport.hooks.install_named_hook("post_connect", calls.append, None) t = self._get_connected_transport() self.assertLength(0, calls) t._set_connection("connection", "auth") self.assertEqual(calls, [t]) class PathFilteringDecoratorTransportTest(tests.TestCase): """Pathfilter decoration specific tests.""" def test_abspath(self): # The abspath is always relative to the base of the backing transport. server = pathfilter.PathFilteringServer( transport.get_transport_from_url("memory:///foo/bar/"), lambda x: x ) server.start_server() t = transport.get_transport_from_url(server.get_url()) self.assertEqual(server.get_url(), t.abspath("/")) subdir_t = t.clone("subdir") self.assertEqual(server.get_url(), subdir_t.abspath("/")) server.stop_server() def make_pf_transport(self, filter_func=None): """Make a PathFilteringTransport backed by a MemoryTransport. :param filter_func: by default this will be a no-op function. Use this parameter to override it. """ if filter_func is None: def filter_func(x): return x server = pathfilter.PathFilteringServer( transport.get_transport_from_url("memory:///foo/bar/"), filter_func ) server.start_server() self.addCleanup(server.stop_server) return transport.get_transport_from_url(server.get_url()) def test__filter(self): # _filter (with an identity func as filter_func) always returns # paths relative to the base of the backing transport. t = self.make_pf_transport() self.assertEqual("foo", t._filter("foo")) self.assertEqual("foo/bar", t._filter("foo/bar")) self.assertEqual("", t._filter("..")) self.assertEqual("", t._filter("/")) # The base of the pathfiltering transport is taken into account too. t = t.clone("subdir1/subdir2") self.assertEqual("subdir1/subdir2/foo", t._filter("foo")) self.assertEqual("subdir1/subdir2/foo/bar", t._filter("foo/bar")) self.assertEqual("subdir1", t._filter("..")) self.assertEqual("", t._filter("/")) def test_filter_invocation(self): filter_log = [] def filter(path): filter_log.append(path) return path t = self.make_pf_transport(filter) t.has("abc") self.assertEqual(["abc"], filter_log) del filter_log[:] t.clone("abc").has("xyz") self.assertEqual(["abc/xyz"], filter_log) del filter_log[:] t.has("/abc") self.assertEqual(["abc"], filter_log) def test_clone(self): t = self.make_pf_transport() # relpath from root and root path are the same relpath_cloned = t.clone("foo") abspath_cloned = t.clone("/foo") self.assertEqual(t.server, relpath_cloned.server) self.assertEqual(t.server, abspath_cloned.server) def test_url_preserves_pathfiltering(self): """Calling get_transport on a pathfiltered transport's base should produce a transport with exactly the same behaviour as the original pathfiltered transport. This is so that it is not possible to escape (accidentally or otherwise) the filtering by doing:: url = filtered_transport.base parent_url = urlutils.join(url, '..') new_t = transport.get_transport_from_url(parent_url) """ t = self.make_pf_transport() new_t = transport.get_transport_from_url(t.base) self.assertEqual(t.server, new_t.server) self.assertEqual(t.base, new_t.base) class ReadonlyDecoratorTransportTest(tests.TestCase): """Readonly decoration specific tests.""" def test_local_parameters(self): # connect to . in readonly mode t = readonly.ReadonlyTransportDecorator("readonly+.") self.assertEqual(True, t.listable()) self.assertEqual(True, t.is_readonly()) def test_http_parameters(self): from .http_server import HttpServer # connect to '.' via http which is not listable server = HttpServer() self.start_server(server) t = transport.get_transport_from_url("readonly+" + server.get_url()) self.assertIsInstance(t, readonly.ReadonlyTransportDecorator) self.assertEqual(False, t.listable()) self.assertEqual(True, t.is_readonly()) class FakeNFSDecoratorTests(tests.TestCaseWithTransport): """NFS decorator specific tests.""" def get_nfs_transport(self, url): # connect to url with nfs decoration return fakenfs.FakeNFSTransportDecorator("fakenfs+" + url) def test_local_parameters(self): # the listable and is_readonly parameters # are not changed by the fakenfs decorator t = self.get_nfs_transport(".") self.assertEqual(True, t.listable()) self.assertEqual(False, t.is_readonly()) def test_http_parameters(self): # the listable and is_readonly parameters # are not changed by the fakenfs decorator from .http_server import HttpServer # connect to '.' via http which is not listable server = HttpServer() self.start_server(server) t = self.get_nfs_transport(server.get_url()) self.assertIsInstance(t, fakenfs.FakeNFSTransportDecorator) self.assertEqual(False, t.listable()) self.assertEqual(True, t.is_readonly()) def test_fakenfs_server_default(self): # a FakeNFSServer() should bring up a local relpath server for itself server = test_server.FakeNFSServer() self.start_server(server) # the url should be decorated appropriately self.assertStartsWith(server.get_url(), "fakenfs+") # and we should be able to get a transport for it t = transport.get_transport_from_url(server.get_url()) # which must be a FakeNFSTransportDecorator instance. self.assertIsInstance(t, fakenfs.FakeNFSTransportDecorator) def test_fakenfs_rename_semantics(self): # a FakeNFS transport must mangle the way rename errors occur to # look like NFS problems. t = self.get_nfs_transport(".") t.mkdir("from") t.put_bytes("from/foo", b"") t.mkdir("to") t.put_bytes("to/bar", b"") self.assertRaises(errors.ResourceBusy, t.rename, "from", "to") class FakeVFATDecoratorTests(tests.TestCaseWithTransport): """Tests for simulation of VFAT restrictions.""" def get_vfat_transport(self, url): """Return vfat-backed transport for test directory.""" from ..fakevfat import FakeVFATTransportDecorator return FakeVFATTransportDecorator("vfat+" + url) def test_transport_creation(self): from ..fakevfat import FakeVFATTransportDecorator t = self.get_vfat_transport(".") self.assertIsInstance(t, FakeVFATTransportDecorator) def test_transport_mkdir(self): t = self.get_vfat_transport(".") t.mkdir("HELLO") self.assertTrue(t.has("hello")) self.assertTrue(t.has("Hello")) def test_forbidden_chars(self): t = self.get_vfat_transport(".") self.assertRaises(ValueError, t.has, "") class BadTransportHandler(transport.Transport): def __init__(self, base_url): raise errors.DependencyNotPresent("some_lib", "testing missing dependency") class BackupTransportHandler(transport.Transport): """Test transport that works as a backup for the BadTransportHandler.""" pass class TestTransportImplementation(tests.TestCaseInTempDir): """Implementation verification for transports. To verify a transport we need a server factory, which is a callable that accepts no parameters and returns an implementation of breezy.transport.Server. That Server is then used to construct transport instances and test the transport via loopback activity. Currently this assumes that the Transport object is connected to the current working directory. So that whatever is done through the transport, should show up in the working directory, and vice-versa. This is a bug, because its possible to have URL schemes which provide access to something that may not be result in storage on the local disk, i.e. due to file system limits, or due to it being a database or some other non-filesystem tool. This also tests to make sure that the functions work with both generators and lists (assuming iter(list) is effectively a generator) """ def setUp(self): super().setUp() self._server = self.transport_server() self.start_server(self._server) def get_transport(self, relpath=None): """Return a connected transport to the local directory. :param relpath: a path relative to the base url. """ base_url = self._server.get_url() url = self._adjust_url(base_url, relpath) # try getting the transport via the regular interface: t = transport.get_transport_from_url(url) # vila--20070607 if the following are commented out the test suite # still pass. Is this really still needed or was it a forgotten # temporary fix ? if not isinstance(t, self.transport_class): # we did not get the correct transport class type. Override the # regular connection behaviour by direct construction. t = self.transport_class(url) return t def build_tree(self, shape, transport=None, line_endings="binary"): """Build a tree of files via the test transport. Transport implementation tests need to operate on files at the test server (which may not be a local filesystem) rather than the process cwd, so this overrides the cwd-based TestCaseInTempDir version. If `transport` is None or read-only, falls back to a transport on the current working directory. Names are URL-escaped before being passed to the transport. """ if transport is None or transport.is_readonly(): from dromedary import get_transport_from_path transport = get_transport_from_path(".") for name in shape: escaped = urlutils.escape(name.rstrip("/")) if name.endswith("/"): transport.mkdir(escaped) else: if line_endings == "binary": end = b"\n" elif line_endings == "native": end = os.linesep.encode("ascii") else: raise ValueError(f"Invalid line ending request {line_endings!r}") content = b"contents of %s%s" % (name.encode("utf-8"), end) transport.put_bytes(escaped, content) class TestTransportFromPath(tests.TestCaseInTempDir): def test_with_path(self): t = transport.get_transport_from_path(self.test_dir) self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base.rstrip("/"), urlutils.local_path_to_url(self.test_dir)) def test_with_url(self): t = transport.get_transport_from_path("file:") self.assertIsInstance(t, local.LocalTransport) self.assertEqual( t.base.rstrip("/"), urlutils.local_path_to_url(os.path.join(self.test_dir, "file:")), ) class TestTransportFromUrl(tests.TestCaseInTempDir): def test_with_path(self): self.assertRaises( urlutils.InvalidURL, transport.get_transport_from_url, self.test_dir ) def test_with_url(self): url = urlutils.local_path_to_url(self.test_dir) t = transport.get_transport_from_url(url) self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base.rstrip("/"), url) def test_with_url_and_segment_parameters(self): url = urlutils.local_path_to_url(self.test_dir) + ",branch=foo" t = transport.get_transport_from_url(url) self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base.rstrip("/"), url) with open(os.path.join(self.test_dir, "afile"), "w") as f: f.write("data") self.assertTrue(t.has("afile")) class TestLocalTransports(tests.TestCase): def test_get_transport_from_abspath(self): here = osutils.abspath(".") t = transport.get_transport_from_path(here) self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base, urlutils.local_path_to_url(here) + "/") def test_get_transport_from_relpath(self): t = transport.get_transport_from_path(".") self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base, urlutils.local_path_to_url(".") + "/") def test_get_transport_from_local_url(self): here = osutils.abspath(".") here_url = urlutils.local_path_to_url(here) + "/" t = transport.get_transport_from_url(here_url) self.assertIsInstance(t, local.LocalTransport) self.assertEqual(t.base, here_url) def test_local_abspath(self): here = osutils.abspath(".") t = transport.get_transport_from_path(here) self.assertEqual(t.local_abspath(""), here) class TestLocalTransportMutation(tests.TestCaseInTempDir): def test_local_transport_mkdir(self): here = osutils.abspath(".") t = transport.get_transport_from_path(here) t.mkdir("test") self.assertTrue(os.path.exists("test")) def test_local_transport_mkdir_permission_denied(self): # See https://bugs.launchpad.net/bzr/+bug/606537 here = osutils.abspath(".") t = transport.get_transport_from_path(here) def fake_chmod(path, mode): e = OSError("permission denied") e.errno = errno.EPERM raise e self.overrideAttr(os, "chmod", fake_chmod) t.mkdir("test") t.mkdir("test2", mode=0o707) self.assertTrue(os.path.exists("test")) self.assertTrue(os.path.exists("test2")) class TestLocalTransportWriteStream(tests.TestCaseWithTransport): def test_local_fdatasync_calls_fdatasync(self): """Check fdatasync on a stream tries to flush the data to the OS. We can't easily observe the external effect but we can at least see it's called. """ sentinel = object() fdatasync = getattr(os, "fdatasync", sentinel) if fdatasync is sentinel: raise tests.TestNotApplicable("fdatasync not supported") t = self.get_transport(".") self.recordCalls(os, "fdatasync") w = t.open_write_stream("out") w.write(b"foo") w.fdatasync() with open("out", "rb") as f: # Should have been flushed. self.assertEqual(f.read(), b"foo") def test_missing_directory(self): t = self.get_transport(".") self.assertRaises(NoSuchFile, t.open_write_stream, "dir/foo") class TestWin32LocalTransport(tests.TestCase): def test_unc_clone_to_root(self): self.requireFeature(features.win32_feature) # Win32 UNC path like \\HOST\path # clone to root should stop at least at \\HOST part # not on \\ t = local.EmulatedWin32LocalTransport("file://HOST/path/to/some/dir/") for _i in range(4): t = t.clone("..") self.assertEqual(t.base, "file://HOST/") # make sure we reach the root t = t.clone("..") self.assertEqual(t.base, "file://HOST/") class TestConnectedTransport(tests.TestCase): """Tests for connected to remote server transports.""" def test_parse_url(self): t = transport.ConnectedTransport("http://simple.example.com/home/source") self.assertEqual(t._parsed_url.host, "simple.example.com") self.assertEqual(t._parsed_url.port, None) self.assertEqual(t._parsed_url.path, "/home/source/") self.assertIsNone(t._parsed_url.user) self.assertIsNone(t._parsed_url.password) self.assertEqual(t.base, "http://simple.example.com/home/source/") def test_parse_url_with_at_in_user(self): # Bug 228058 t = transport.ConnectedTransport("ftp://user@host.com@www.host.com/") self.assertEqual(t._parsed_url.user, "user@host.com") def test_parse_quoted_url(self): t = transport.ConnectedTransport("http://ro%62ey:h%40t@ex%41mple.com:2222/path") self.assertEqual(t._parsed_url.host, "exAmple.com") self.assertEqual(t._parsed_url.port, 2222) self.assertEqual(t._parsed_url.user, "robey") self.assertEqual(t._parsed_url.password, "h@t") self.assertEqual(t._parsed_url.path, "/path/") # Base should not keep track of the password self.assertEqual(t.base, "http://ro%62ey@ex%41mple.com:2222/path/") def test_parse_invalid_url(self): self.assertRaises( urlutils.InvalidURL, transport.ConnectedTransport, "sftp://lily.org:~janneke/public/bzr/gub", ) def test_relpath(self): t = transport.ConnectedTransport("sftp://user@host.com/abs/path") self.assertEqual(t.relpath("sftp://user@host.com/abs/path/sub"), "sub") self.assertRaises( errors.PathNotChild, t.relpath, "http://user@host.com/abs/path/sub" ) self.assertRaises( errors.PathNotChild, t.relpath, "sftp://user2@host.com/abs/path/sub" ) self.assertRaises( errors.PathNotChild, t.relpath, "sftp://user@otherhost.com/abs/path/sub" ) self.assertRaises( errors.PathNotChild, t.relpath, "sftp://user@host.com:33/abs/path/sub" ) # Make sure it works when we don't supply a username t = transport.ConnectedTransport("sftp://host.com/abs/path") self.assertEqual(t.relpath("sftp://host.com/abs/path/sub"), "sub") # Make sure it works when parts of the path will be url encoded t = transport.ConnectedTransport("sftp://host.com/dev/%path") self.assertEqual(t.relpath("sftp://host.com/dev/%path/sub"), "sub") def test_connection_sharing_propagate_credentials(self): t = transport.ConnectedTransport("ftp://user@host.com/abs/path") self.assertEqual("user", t._parsed_url.user) self.assertEqual("host.com", t._parsed_url.host) self.assertIs(None, t._get_connection()) self.assertIs(None, t._parsed_url.password) c = t.clone("subdir") self.assertIs(None, c._get_connection()) self.assertIs(None, t._parsed_url.password) # Simulate the user entering a password password = "secret" connection = object() t._set_connection(connection, password) self.assertIs(connection, t._get_connection()) self.assertIs(password, t._get_credentials()) self.assertIs(connection, c._get_connection()) self.assertIs(password, c._get_credentials()) # credentials can be updated new_password = "even more secret" c._update_credentials(new_password) self.assertIs(connection, t._get_connection()) self.assertIs(new_password, t._get_credentials()) self.assertIs(connection, c._get_connection()) self.assertIs(new_password, c._get_credentials()) class TestReusedTransports(tests.TestCase): """Tests for transport reuse.""" def test_reuse_same_transport(self): possible_transports = [] t1 = transport.get_transport_from_url( "http://foo/", possible_transports=possible_transports ) self.assertEqual([t1], possible_transports) t2 = transport.get_transport_from_url("http://foo/", possible_transports=[t1]) self.assertIs(t1, t2) # Also check that final '/' are handled correctly t3 = transport.get_transport_from_url("http://foo/path/") t4 = transport.get_transport_from_url( "http://foo/path", possible_transports=[t3] ) self.assertIs(t3, t4) t5 = transport.get_transport_from_url("http://foo/path") t6 = transport.get_transport_from_url( "http://foo/path/", possible_transports=[t5] ) self.assertIs(t5, t6) def test_don_t_reuse_different_transport(self): t1 = transport.get_transport_from_url("http://foo/path") t2 = transport.get_transport_from_url( "http://bar/path", possible_transports=[t1] ) self.assertIsNot(t1, t2) class TestTransportTrace(tests.TestCase): def test_decorator(self): t = transport.get_transport_from_url("trace+memory://") from dromedary.trace import TransportTraceDecorator self.assertIsInstance(t, TransportTraceDecorator) def test_clone_preserves_activity(self): t = transport.get_transport_from_url("trace+memory://") t2 = t.clone(".") self.assertIsNot(t, t2) self.assertIs(t._activity, t2._activity) # the following specific tests are for the operations that have made use of # logging in tests; we could test every single operation but doing that # still won't cause a test failure when the top level Transport API # changes; so there is little return doing that. def test_get(self): t = transport.get_transport_from_url("trace+memory:///") t.put_bytes("foo", b"barish") t.get("foo") expected_result = [] # put_bytes records the bytes, not the content to avoid memory # pressure. expected_result.append(("put_bytes", "foo", 6, None)) # get records the file name only. expected_result.append(("get", "foo")) self.assertEqual(expected_result, t._activity) def test_readv(self): t = transport.get_transport_from_url("trace+memory:///") t.put_bytes("foo", b"barish") list(t.readv("foo", [(0, 1), (3, 2)], adjust_for_latency=True, upper_limit=6)) expected_result = [] # put_bytes records the bytes, not the content to avoid memory # pressure. expected_result.append(("put_bytes", "foo", 6, None)) # readv records the supplied offset request expected_result.append(("readv", "foo", [(0, 1), (3, 2)], True, 6)) self.assertEqual(expected_result, t._activity) class TestSSHConnections(tests.TestCaseWithTransport): def test_bzr_connect_to_bzr_ssh(self): """get_transport of a bzr+ssh:// behaves correctly. bzr+ssh:// should cause bzr to run a remote bzr smart server over SSH. Note: this test requires breezy's bzr+ssh transport to be registered. """ raise tests.TestNotApplicable( "bzr+ssh:// is registered by breezy, not dromedary" ) # This test actually causes a bzr instance to be invoked, which is very # expensive: it should be the only such test in the test suite. # A reasonable evolution for this would be to simply check inside # check_channel_exec_request that the command is appropriate, and then # satisfy requests in-process. self.requireFeature(features.paramiko) # SFTPFullAbsoluteServer has a get_url method, and doesn't # override the interface (doesn't change self._vendor). # Note that this does encryption, so can be slow. from dromedary.tests import stub_sftp # Start an SSH server self.command_executed = [] # XXX: This is horrible -- we define a really dumb SSH server that # executes commands, and manage the hooking up of stdin/out/err to the # SSH channel ourselves. Surely this has already been implemented # elsewhere? started = [] class StubSSHServer(stub_sftp.StubServer): test = self def check_channel_exec_request(self, channel, command): self.test.command_executed.append(command) proc = subprocess.Popen( command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, ) # XXX: horribly inefficient, not to mention ugly. # Start a thread for each of stdin/out/err, and relay bytes # from the subprocess to channel and vice versa. def ferry_bytes(read, write, close): while True: bytes = read(1) if bytes == b"": close() break write(bytes) file_functions = [ (channel.recv, proc.stdin.write, proc.stdin.close), (proc.stdout.read, channel.sendall, channel.close), (proc.stderr.read, channel.sendall_stderr, channel.close), ] started.append(proc) for read, write, close in file_functions: t = threading.Thread(target=ferry_bytes, args=(read, write, close)) t.start() started.append(t) return True ssh_server = stub_sftp.SFTPFullAbsoluteServer(StubSSHServer) # We *don't* want to override the default SSH vendor: the detected one # is the one to use. # FIXME: I don't understand the above comment, SFTPFullAbsoluteServer # inherits from SFTPServer which forces the SSH vendor to # ssh.ParamikoVendor(). So it's forced, not detected. --vila 20100623 self.start_server(ssh_server) port = ssh_server.port bzr_remote_command = self.get_brz_command() self.overrideEnv("BZR_REMOTE_PATH", " ".join(bzr_remote_command)) self.overrideEnv("PYTHONPATH", ":".join(sys.path)) # Access the branch via a bzr+ssh URL. The BZR_REMOTE_PATH environment # variable is used to tell bzr what command to run on the remote end. path_to_branch = osutils.abspath(".") if sys.platform == "win32": # On Windows, we export all drives as '/C:/, etc. So we need to # prefix a '/' to get the right path. path_to_branch = "/" + path_to_branch url = "bzr+ssh://fred:secret@localhost:%d%s" % (port, path_to_branch) t = transport.get_transport_from_url(url) self.permit_url(t.base) t.mkdir("foo") self.assertEqual( [ b"%s serve --inet --directory=/ --allow-writes" % " ".join(bzr_remote_command).encode() ], self.command_executed, ) # Make sure to disconnect, so that the remote process can stop, and we # can cleanup. Then pause the test until everything is shutdown t._client._medium.disconnect() if not started: return # First wait for the subprocess started[0].wait() # And the rest are threads for t in started[1:]: t.join() class TestKind(tests.TestCaseWithTransport): def test_file_kind(self): import socket self.build_tree(["file", "dir/"]) self.assertEqual("file", file_kind("file")) self.assertEqual("directory", file_kind("dir/")) if osutils.supports_symlinks(self.test_dir): os.symlink("symlink", "symlink") self.assertEqual("symlink", file_kind("symlink")) # TODO: jam 20060529 Test a block device try: os.lstat("/dev/null") except FileNotFoundError: pass else: self.assertEqual("chardev", file_kind(os.path.realpath("/dev/null"))) mkfifo = getattr(os, "mkfifo", None) if mkfifo: mkfifo("fifo") try: self.assertEqual("fifo", file_kind("fifo")) finally: os.remove("fifo") AF_UNIX = getattr(socket, "AF_UNIX", None) if AF_UNIX: s = socket.socket(AF_UNIX) s.bind("socket") try: self.assertEqual("socket", file_kind("socket")) finally: os.remove("socket") dromedary-0.1.1/tests/test_transport_log.py000064400000000000000000000054371046102023000172720ustar 00000000000000# Copyright (C) 2008-2011, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Tests for log+ transport decorator.""" import logging from dromedary.tests import TestCaseWithMemoryTransport logger = logging.getLogger("dromedary.tests.test_transport_log") import dromedary as transport from dromedary.log import TransportLogDecorator class TestTransportLog(TestCaseWithMemoryTransport): def test_log_transport(self): base_transport = self.get_transport("") logging_transport = transport.get_transport_from_url( "log+" + base_transport.base ) # operations such as mkdir are logged logger.debug("where are you?") logging_transport.mkdir("subdir") log = self.get_log() # GZ 2017-05-24: Used to expect abspath logged, logger needs fixing. self.assertContainsRe(log, r"mkdir subdir") self.assertContainsRe(log, " --> None") # they have the expected effect self.assertTrue(logging_transport.has("subdir")) # and they operate on the underlying transport self.assertTrue(base_transport.has("subdir")) def test_log_readv(self): # see # transports are not required to return a generator, but we # specifically want to check that those that do cause it to be passed # through, for the sake of minimum interference base_transport = DummyReadvTransport() # construct it directly to avoid needing the dummy transport to be # registered etc logging_transport = TransportLogDecorator( "log+dummy:///", _decorated=base_transport ) result = base_transport.readv("foo", [(0, 10)]) # sadly there's no types.IteratorType, and GeneratorType is too # specific next(result) result = logging_transport.readv("foo", [(0, 10)]) self.assertEqual(list(result), [(0, "abcdefghij")]) class DummyReadvTransport: base = "dummy:///" def readv(self, filename, offset_length_pairs): yield (0, "abcdefghij") def abspath(self, path): return self.base + path dromedary-0.1.1/tests/test_urlutils.py000064400000000000000000001412611046102023000162540ustar 00000000000000# Copyright (C) 2006-2012, 2015, 2016 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Tests for the urlutils wrapper.""" import ntpath import os import posixpath import sys from dromedary import osutils, urlutils from dromedary import tests as features from dromedary.errors import PathNotChild from dromedary.tests import TestCase, TestCaseInTempDir, TestSkipped class TestUrlToPath(TestCase): def test_basename(self): # breezy.urlutils.basename # Test breezy.urlutils.split() basename = urlutils.basename if sys.platform == "win32": self.assertRaises(urlutils.InvalidURL, basename, "file:///path/to/foo") self.assertEqual("foo", basename("file:///C|/foo")) self.assertEqual("foo", basename("file:///C:/foo")) self.assertEqual("", basename("file:///C:/")) else: self.assertEqual("foo", basename("file:///foo")) self.assertEqual("", basename("file:///")) self.assertEqual("foo", basename("http://host/path/to/foo")) self.assertEqual("foo", basename("http://host/path/to/foo/")) self.assertEqual( "", basename("http://host/path/to/foo/", exclude_trailing_slash=False) ) self.assertEqual("path", basename("http://host/path")) self.assertEqual("", basename("http://host/")) self.assertEqual("", basename("http://host")) self.assertEqual("path", basename("http:///nohost/path")) self.assertEqual("path", basename("random+scheme://user:pass@ahost:port/path")) self.assertEqual("path", basename("random+scheme://user:pass@ahost:port/path/")) self.assertEqual("", basename("random+scheme://user:pass@ahost:port/")) # relative paths self.assertEqual("foo", basename("path/to/foo")) self.assertEqual("foo", basename("path/to/foo/")) self.assertEqual("", basename("path/to/foo/", exclude_trailing_slash=False)) self.assertEqual("foo", basename("path/../foo")) self.assertEqual("foo", basename("../path/foo")) def test_normalize_url_files(self): # Test that local paths are properly normalized normalize_url = urlutils.normalize_url def norm_file(expected, path): url = normalize_url(path) self.assertStartsWith(url, "file:///") if sys.platform == "win32": url = url[len("file:///C:") :] else: url = url[len("file://") :] self.assertEndsWith(url, expected) norm_file("path/to/foo", "path/to/foo") norm_file("/path/to/foo", "/path/to/foo") norm_file("path/to/foo", "../path/to/foo") # Local paths are assumed to *not* be escaped at all try: "uni/\xb5".encode(osutils.get_user_encoding()) except UnicodeError: # locale cannot handle unicode pass else: norm_file("uni/%C2%B5", "uni/\xb5") norm_file("uni/%25C2%25B5", "uni/%C2%B5") norm_file("uni/%20b", "uni/ b") # All the crazy characters get escaped in local paths => file:/// urls # The ' ' character must not be at the end, because on win32 # it gets stripped off by ntpath.abspath norm_file("%27%20%3B/%3F%3A%40%26%3D%2B%24%2C%23", "' ;/?:@&=+$,#") def test_normalize_url_hybrid(self): # Anything with a scheme:// should be treated as a hybrid url # which changes what characters get escaped. normalize_url = urlutils.normalize_url eq = self.assertEqual eq("file:///foo/", normalize_url("file:///foo/")) eq("file:///foo/%20", normalize_url("file:///foo/ ")) eq("file:///foo/%20", normalize_url("file:///foo/%20")) # Don't escape reserved characters eq( "file:///ab_c.d-e/%f:?g&h=i+j;k,L#M$", normalize_url("file:///ab_c.d-e/%f:?g&h=i+j;k,L#M$"), ) eq( "http://ab_c.d-e/%f:?g&h=i+j;k,L#M$", normalize_url("http://ab_c.d-e/%f:?g&h=i+j;k,L#M$"), ) # Escape unicode characters, but not already escaped chars eq("http://host/ab/%C2%B5/%C2%B5", normalize_url("http://host/ab/%C2%B5/\xb5")) # Unescape characters that don't need to be escaped eq( "http://host/~bob%2525-._", normalize_url("http://host/%7Ebob%2525%2D%2E%5F"), ) eq( "http://host/~bob%2525-._", normalize_url("http://host/%7Ebob%2525%2D%2E%5F"), ) def test_url_scheme_re(self): # Test paths that may be URLs def test_one(url, scheme_and_path): """Assert that _url_scheme_re correctly matches. :param scheme_and_path: The (scheme, path) that should be matched can be None, to indicate it should not match """ (scheme_pos, separator_pos) = urlutils._find_scheme_and_separator(url) if scheme_and_path is None: self.assertIs(None, scheme_pos) self.assertIs(None, separator_pos) else: self.assertEqual(scheme_and_path[0], url[:scheme_pos]) self.assertEqual(scheme_and_path[1], url[separator_pos:]) # Local paths test_one("/path", None) test_one("C:/path", None) test_one("../path/to/foo", None) test_one("../path/to/fo\xe5", None) # Real URLS test_one("http://host/path/", ("http", "/path/")) test_one("sftp://host/path/to/foo", ("sftp", "/path/to/foo")) test_one("file:///usr/bin", ("file", "/usr/bin")) test_one("file:///C:/Windows", ("file", "/C:/Windows")) test_one("file:///C|/Windows", ("file", "/C|/Windows")) test_one("readonly+sftp://host/path/\xe5", ("readonly+sftp", "/path/\xe5")) # Weird stuff # Can't have slashes or colons in the scheme test_one("/path/to/://foo", None) test_one("scheme:stuff://foo", ("scheme", "//foo")) # Must have more than one character for scheme test_one("C://foo", None) test_one("ab://foo", ("ab", "ab://foo")) def test_dirname(self): # Test breezy.urlutils.dirname() dirname = urlutils.dirname if sys.platform == "win32": self.assertRaises(urlutils.InvalidURL, dirname, "file:///path/to/foo") self.assertEqual("file:///C|/", dirname("file:///C|/foo")) self.assertEqual("file:///C|/", dirname("file:///C|/")) else: self.assertEqual("file:///", dirname("file:///foo")) self.assertEqual("file:///", dirname("file:///")) self.assertEqual("http://host/path/to", dirname("http://host/path/to/foo")) self.assertEqual("http://host/path/to", dirname("http://host/path/to/foo/")) self.assertEqual( "http://host/path/to/foo", dirname("http://host/path/to/foo/", exclude_trailing_slash=False), ) self.assertEqual("http://host/", dirname("http://host/path")) self.assertEqual("http://host/", dirname("http://host/")) self.assertEqual("http://host", dirname("http://host")) self.assertEqual("http:///nohost", dirname("http:///nohost/path")) self.assertEqual( "random+scheme://user:pass@ahost:port/", dirname("random+scheme://user:pass@ahost:port/path"), ) self.assertEqual( "random+scheme://user:pass@ahost:port/", dirname("random+scheme://user:pass@ahost:port/path/"), ) self.assertEqual( "random+scheme://user:pass@ahost:port/", dirname("random+scheme://user:pass@ahost:port/"), ) # relative paths self.assertEqual("path/to", dirname("path/to/foo")) self.assertEqual("path/to", dirname("path/to/foo/")) self.assertEqual( "path/to/foo", dirname("path/to/foo/", exclude_trailing_slash=False) ) self.assertEqual("path/..", dirname("path/../foo")) self.assertEqual("../path", dirname("../path/foo")) def test_is_url(self): self.assertTrue(urlutils.is_url("http://foo/bar")) self.assertTrue(urlutils.is_url("bzr+ssh://foo/bar")) self.assertTrue(urlutils.is_url("lp:foo/bar")) self.assertTrue(urlutils.is_url("file:///foo/bar")) self.assertFalse(urlutils.is_url("")) self.assertFalse(urlutils.is_url("foo")) self.assertFalse(urlutils.is_url("foo/bar")) self.assertFalse(urlutils.is_url("/foo")) self.assertFalse(urlutils.is_url("/foo/bar")) self.assertFalse(urlutils.is_url("C:/")) self.assertFalse(urlutils.is_url("C:/foo")) self.assertFalse(urlutils.is_url("C:/foo/bar")) def test_join(self): def test(expected, *args): joined = urlutils.join(*args) self.assertEqual(expected, joined) # Test relative path joining test("foo", "foo") # relative fragment with nothing is preserved. test("foo/bar", "foo", "bar") test("http://foo/bar", "http://foo", "bar") test("http://foo/bar", "http://foo", ".", "bar") test("http://foo/baz", "http://foo", "bar", "../baz") test("http://foo/bar/baz", "http://foo", "bar/baz") test("http://foo/baz", "http://foo", "bar/../baz") test("http://foo/baz", "http://foo/bar/", "../baz") test("lp:foo/bar", "lp:foo", "bar") test("lp:foo/bar/baz", "lp:foo", "bar/baz") # Absolute paths test("http://foo", "http://foo") # abs url with nothing is preserved. test("http://bar", "http://foo", "http://bar") test("sftp://bzr/foo", "http://foo", "bar", "sftp://bzr/foo") test("file:///bar", "foo", "file:///bar") test("http://bar/", "http://foo", "http://bar/") test("http://bar/a", "http://foo", "http://bar/a") test("http://bar/a/", "http://foo", "http://bar/a/") test("lp:bar", "http://foo", "lp:bar") test("lp:bar", "lp:foo", "lp:bar") test("file:///stuff", "lp:foo", "file:///stuff") # From a base path test("file:///foo", "file:///", "foo") test("file:///bar/foo", "file:///bar/", "foo") test("http://host/foo", "http://host/", "foo") test("http://host/", "http://host", "") # Invalid joinings # Cannot go above root # Implicitly at root: self.assertRaises( urlutils.InvalidURLJoin, urlutils.join, "http://foo", "../baz" ) self.assertRaises(urlutils.InvalidURLJoin, urlutils.join, "http://foo", "/..") # Joining from a path explicitly under the root. self.assertRaises( urlutils.InvalidURLJoin, urlutils.join, "http://foo/a", "../../b" ) def test_joinpath(self): def test(expected, *args): joined = urlutils.joinpath(*args) self.assertEqual(expected, joined) # Test a single element test("foo", "foo") # Test relative path joining test("foo/bar", "foo", "bar") test("foo/bar", "foo", ".", "bar") test("foo/baz", "foo", "bar", "../baz") test("foo/bar/baz", "foo", "bar/baz") test("foo/baz", "foo", "bar/../baz") # Test joining to an absolute path test("/foo", "/foo") test("/foo", "/foo", ".") test("/foo/bar", "/foo", "bar") test("/", "/foo", "..") # Test joining with an absolute path test("/bar", "foo", "/bar") # Test joining to a path with a trailing slash test("foo/bar", "foo/", "bar") # Invalid joinings # Cannot go above root self.assertRaises(urlutils.InvalidURLJoin, urlutils.joinpath, "/", "../baz") self.assertRaises(urlutils.InvalidURLJoin, urlutils.joinpath, "/", "..") self.assertRaises(urlutils.InvalidURLJoin, urlutils.joinpath, "/", "/..") def test_join_segment_parameters_raw(self): join_segment_parameters_raw = urlutils.join_segment_parameters_raw self.assertEqual("/somedir/path", join_segment_parameters_raw("/somedir/path")) self.assertEqual( "/somedir/path,rawdata", join_segment_parameters_raw("/somedir/path", "rawdata"), ) self.assertRaises( urlutils.InvalidURLJoin, join_segment_parameters_raw, "/somedir/path", "rawdata1,rawdata2,rawdata3", ) self.assertEqual( "/somedir/path,bla,bar", join_segment_parameters_raw("/somedir/path", "bla", "bar"), ) self.assertEqual( "/somedir,exist=some/path,bla,bar", join_segment_parameters_raw("/somedir,exist=some/path", "bla", "bar"), ) self.assertRaises(TypeError, join_segment_parameters_raw, "/somepath", 42) def test_join_segment_parameters(self): join_segment_parameters = urlutils.join_segment_parameters self.assertEqual("/somedir/path", join_segment_parameters("/somedir/path", {})) self.assertEqual( "/somedir/path,key1=val1", join_segment_parameters("/somedir/path", {"key1": "val1"}), ) self.assertRaises( urlutils.InvalidURLJoin, join_segment_parameters, "/somedir/path", {"branch": "brr,brr,brr"}, ) self.assertRaises( urlutils.InvalidURLJoin, join_segment_parameters, "/somedir/path", {"key1=val1": "val2"}, ) self.assertEqual( "/somedir/path,key1=val1,key2=val2", join_segment_parameters("/somedir/path", {"key1": "val1", "key2": "val2"}), ) self.assertEqual( "/somedir/path,key1=val1,key2=val2", join_segment_parameters("/somedir/path,key1=val1", {"key2": "val2"}), ) self.assertEqual( "/somedir/path,key1=val2", join_segment_parameters("/somedir/path,key1=val1", {"key1": "val2"}), ) self.assertEqual( "/somedir,exist=some/path,key1=val1", join_segment_parameters("/somedir,exist=some/path", {"key1": "val1"}), ) self.assertEqual( "/,key1=val1,key2=val2", join_segment_parameters("/,key1=val1", {"key2": "val2"}), ) self.assertRaises( TypeError, join_segment_parameters, "/,key1=val1", {"foo": 42} ) def test_posix_local_path_to_url(self): to_url = urlutils._posix_local_path_to_url self.assertEqual("file:///path/to/foo", to_url("/path/to/foo")) self.assertEqual("file:///path/to/foo%2Cbar", to_url("/path/to/foo,bar")) try: result = to_url("/path/to/r\xe4ksm\xf6rg\xe5s") except UnicodeError as err: raise TestSkipped("local encoding cannot handle unicode") from err self.assertEqual("file:///path/to/r%C3%A4ksm%C3%B6rg%C3%A5s", result) self.assertIsInstance(result, str) def test_posix_local_path_from_url(self): from_url = urlutils._posix_local_path_from_url self.assertEqual("/path/to/foo", from_url("file:///path/to/foo")) self.assertEqual("/path/to/foo", from_url("file:///path/to/foo,branch=foo")) self.assertEqual( "/path/to/r\xe4ksm\xf6rg\xe5s", from_url("file:///path/to/r%C3%A4ksm%C3%B6rg%C3%A5s"), ) self.assertEqual( "/path/to/r\xe4ksm\xf6rg\xe5s", from_url("file:///path/to/r%c3%a4ksm%c3%b6rg%c3%a5s"), ) self.assertEqual( "/path/to/r\xe4ksm\xf6rg\xe5s", from_url("file://localhost/path/to/r%c3%a4ksm%c3%b6rg%c3%a5s"), ) self.assertRaises(urlutils.InvalidURL, from_url, "/path/to/foo") self.assertRaises( urlutils.InvalidURL, from_url, "file://remotehost/path/to/r%c3%a4ksm%c3%b6rg%c3%a5s", ) def test_win32_local_path_to_url(self): to_url = urlutils._win32_local_path_to_url self.assertEqual("file:///C:/path/to/foo", to_url("C:/path/to/foo")) # BOGUS: on win32, ntpath.abspath will strip trailing # whitespace, so this will always fail # Though under linux, it fakes abspath support # and thus will succeed # self.assertEqual('file:///C:/path/to/foo%20', # to_url('C:/path/to/foo ')) self.assertEqual("file:///C:/path/to/f%20oo", to_url("C:/path/to/f oo")) self.assertEqual("file:///", to_url("/")) self.assertEqual("file:///C:/path/to/foo%2Cbar", to_url("C:/path/to/foo,bar")) try: result = to_url("d:/path/to/r\xe4ksm\xf6rg\xe5s") except UnicodeError as err: raise TestSkipped("local encoding cannot handle unicode") from err self.assertEqual("file:///D:/path/to/r%C3%A4ksm%C3%B6rg%C3%A5s", result) self.assertIsInstance(result, str) def test_win32_unc_path_to_url(self): self.requireFeature(features.win32_feature) to_url = urlutils._win32_local_path_to_url self.assertEqual("file://HOST/path", to_url(r"\\HOST\path")) self.assertEqual("file://HOST/path", to_url("//HOST/path")) try: result = to_url("//HOST/path/to/r\xe4ksm\xf6rg\xe5s") except UnicodeError as err: raise TestSkipped("local encoding cannot handle unicode") from err self.assertEqual("file://HOST/path/to/r%C3%A4ksm%C3%B6rg%C3%A5s", result) self.assertNotIsInstance(result, str) def test_win32_local_path_from_url(self): from_url = urlutils._win32_local_path_from_url self.assertEqual("C:/path/to/foo", from_url("file:///C|/path/to/foo")) self.assertEqual( "D:/path/to/r\xe4ksm\xf6rg\xe5s", from_url("file:///d|/path/to/r%C3%A4ksm%C3%B6rg%C3%A5s"), ) self.assertEqual( "D:/path/to/r\xe4ksm\xf6rg\xe5s", from_url("file:///d:/path/to/r%c3%a4ksm%c3%b6rg%c3%a5s"), ) self.assertEqual("/", from_url("file:///")) self.assertEqual( "C:/path/to/foo", from_url("file:///C|/path/to/foo,branch=foo") ) self.assertRaises(urlutils.InvalidURL, from_url, "file:///C:") self.assertRaises(urlutils.InvalidURL, from_url, "file:///c") self.assertRaises(urlutils.InvalidURL, from_url, "/path/to/foo") # Not a valid _win32 url, no drive letter self.assertRaises(urlutils.InvalidURL, from_url, "file:///path/to/foo") def test_win32_unc_path_from_url(self): from_url = urlutils._win32_local_path_from_url self.assertEqual("//HOST/path", from_url("file://HOST/path")) self.assertEqual("//HOST/path", from_url("file://HOST/path,branch=foo")) # despite IE allows 2, 4, 5 and 6 slashes in URL to another machine # we want to use only 2 slashes # Firefox understand only 5 slashes in URL, but it's ugly self.assertRaises(urlutils.InvalidURL, from_url, "file:////HOST/path") self.assertRaises(urlutils.InvalidURL, from_url, "file://///HOST/path") self.assertRaises(urlutils.InvalidURL, from_url, "file://////HOST/path") # check for file://C:/ instead of file:///C:/ self.assertRaises(urlutils.InvalidURL, from_url, "file://C:/path") def test_win32_extract_drive_letter(self): extract = urlutils._win32_extract_drive_letter self.assertEqual(("file:///C:", "/foo"), extract("file://", "/C:/foo")) self.assertEqual(("file:///d|", "/path"), extract("file://", "/d|/path")) self.assertRaises(urlutils.InvalidURL, extract, "file://", "/path") # Root drives without slash treated as invalid, see bug #841322 self.assertEqual(("file:///C:", "/"), extract("file://", "/C:/")) self.assertRaises(urlutils.InvalidURL, extract, "file://", "/C:") # Invalid without drive separator or following forward slash self.assertRaises(urlutils.InvalidURL, extract, "file://", "/C") self.assertRaises(urlutils.InvalidURL, extract, "file://", "/C:ool") def test_split(self): # Test breezy.urlutils.split() split = urlutils.split if sys.platform == "win32": self.assertRaises(urlutils.InvalidURL, split, "file:///path/to/foo") self.assertEqual(("file:///C|/", "foo"), split("file:///C|/foo")) self.assertEqual(("file:///C:/", ""), split("file:///C:/")) else: self.assertEqual(("file:///", "foo"), split("file:///foo")) self.assertEqual(("file:///", ""), split("file:///")) self.assertEqual( ("http://host/path/to", "foo"), split("http://host/path/to/foo") ) self.assertEqual( ("http://host/path/to", "foo"), split("http://host/path/to/foo/") ) self.assertEqual( ("http://host/path/to/foo", ""), split("http://host/path/to/foo/", exclude_trailing_slash=False), ) self.assertEqual(("http://host/", "path"), split("http://host/path")) self.assertEqual(("http://host/", ""), split("http://host/")) self.assertEqual(("http://host", ""), split("http://host")) self.assertEqual(("http:///nohost", "path"), split("http:///nohost/path")) self.assertEqual( ("random+scheme://user:pass@ahost:port/", "path"), split("random+scheme://user:pass@ahost:port/path"), ) self.assertEqual( ("random+scheme://user:pass@ahost:port/", "path"), split("random+scheme://user:pass@ahost:port/path/"), ) self.assertEqual( ("random+scheme://user:pass@ahost:port/", ""), split("random+scheme://user:pass@ahost:port/"), ) # relative paths self.assertEqual(("path/to", "foo"), split("path/to/foo")) self.assertEqual(("path/to", "foo"), split("path/to/foo/")) self.assertEqual( ("path/to/foo", ""), split("path/to/foo/", exclude_trailing_slash=False) ) self.assertEqual(("path/..", "foo"), split("path/../foo")) self.assertEqual(("../path", "foo"), split("../path/foo")) def test_strip_segment_parameters(self): strip_segment_parameters = urlutils.strip_segment_parameters # Check relative references with absolute paths self.assertEqual("/some/path", strip_segment_parameters("/some/path")) self.assertEqual("/some/path", strip_segment_parameters("/some/path,tip")) self.assertEqual( "/some,dir/path", strip_segment_parameters("/some,dir/path,tip") ) self.assertEqual( "/somedir/path", strip_segment_parameters("/somedir/path,heads%2Ftip") ) self.assertEqual( "/somedir/path", strip_segment_parameters("/somedir/path,heads%2Ftip,bar") ) # Check relative references with relative paths self.assertEqual("", strip_segment_parameters(",key1=val1")) self.assertEqual("foo/", strip_segment_parameters("foo/,key1=val1")) self.assertEqual("foo", strip_segment_parameters("foo,key1=val1")) self.assertEqual( "foo/base,la=bla/other/elements", strip_segment_parameters("foo/base,la=bla/other/elements"), ) self.assertEqual( "foo/base,la=bla/other/elements", strip_segment_parameters("foo/base,la=bla/other/elements,a=b"), ) # TODO: Check full URLs as well as relative references def test_split_segment_parameters_raw(self): split_segment_parameters_raw = urlutils.split_segment_parameters_raw # Check relative references with absolute paths self.assertEqual(("/some/path", []), split_segment_parameters_raw("/some/path")) self.assertEqual( ("/some/path", ["tip"]), split_segment_parameters_raw("/some/path,tip") ) self.assertEqual( ("/some,dir/path", ["tip"]), split_segment_parameters_raw("/some,dir/path,tip"), ) self.assertEqual( ("/somedir/path", ["heads%2Ftip"]), split_segment_parameters_raw("/somedir/path,heads%2Ftip"), ) self.assertEqual( ("/somedir/path", ["heads%2Ftip", "bar"]), split_segment_parameters_raw("/somedir/path,heads%2Ftip,bar"), ) # Check relative references with relative paths self.assertEqual( ("", ["key1=val1"]), split_segment_parameters_raw(",key1=val1") ) self.assertEqual( ("foo/", ["key1=val1"]), split_segment_parameters_raw("foo/,key1=val1") ) self.assertEqual( ("foo", ["key1=val1"]), split_segment_parameters_raw("foo,key1=val1") ) self.assertEqual( ("foo/base,la=bla/other/elements", []), split_segment_parameters_raw("foo/base,la=bla/other/elements"), ) self.assertEqual( ("foo/base,la=bla/other/elements", ["a=b"]), split_segment_parameters_raw("foo/base,la=bla/other/elements,a=b"), ) # TODO: Check full URLs as well as relative references def test_split_segment_parameters(self): split_segment_parameters = urlutils.split_segment_parameters # Check relative references with absolute paths self.assertEqual(("/some/path", {}), split_segment_parameters("/some/path")) self.assertEqual( ("/some/path", {"branch": "tip"}), split_segment_parameters("/some/path,branch=tip"), ) self.assertEqual( ("/some,dir/path", {"branch": "tip"}), split_segment_parameters("/some,dir/path,branch=tip"), ) self.assertEqual( ("/somedir/path", {"ref": "heads%2Ftip"}), split_segment_parameters("/somedir/path,ref=heads%2Ftip"), ) self.assertEqual( ("/somedir/path", {"ref": "heads%2Ftip", "key1": "val1"}), split_segment_parameters("/somedir/path,ref=heads%2Ftip,key1=val1"), ) self.assertEqual( ("/somedir/path", {"ref": "heads%2F=tip"}), split_segment_parameters("/somedir/path,ref=heads%2F=tip"), ) # Check relative references with relative paths self.assertEqual(("", {"key1": "val1"}), split_segment_parameters(",key1=val1")) self.assertEqual( ("foo/", {"key1": "val1"}), split_segment_parameters("foo/,key1=val1") ) self.assertEqual( ("foo/base,key1=val1/other/elements", {}), split_segment_parameters("foo/base,key1=val1/other/elements"), ) self.assertEqual( ("foo/base,key1=val1/other/elements", {"key2": "val2"}), split_segment_parameters("foo/base,key1=val1/other/elements,key2=val2"), ) self.assertRaises( urlutils.InvalidURL, split_segment_parameters, "foo/base,key1" ) # TODO: Check full URLs as well as relative references def test_win32_strip_local_trailing_slash(self): strip = urlutils._win32_strip_local_trailing_slash self.assertEqual("file://", strip("file://")) self.assertEqual("file:///", strip("file:///")) self.assertEqual("file:///C", strip("file:///C")) self.assertEqual("file:///C:", strip("file:///C:")) self.assertEqual("file:///d|", strip("file:///d|")) self.assertEqual("file:///C:/", strip("file:///C:/")) self.assertEqual("file:///C:/a", strip("file:///C:/a/")) def test_strip_trailing_slash(self): sts = urlutils.strip_trailing_slash if sys.platform == "win32": self.assertEqual("file:///C|/", sts("file:///C|/")) self.assertEqual("file:///C:/foo", sts("file:///C:/foo")) self.assertEqual("file:///C|/foo", sts("file:///C|/foo/")) else: self.assertEqual("file:///", sts("file:///")) self.assertEqual("file:///foo", sts("file:///foo")) self.assertEqual("file:///foo", sts("file:///foo/")) self.assertEqual("http://host/", sts("http://host/")) self.assertEqual("http://host/foo", sts("http://host/foo")) self.assertEqual("http://host/foo", sts("http://host/foo/")) # No need to fail just because the slash is missing self.assertEqual("http://host", sts("http://host")) # TODO: jam 20060502 Should this raise InvalidURL? self.assertEqual("file://", sts("file://")) self.assertEqual( "random+scheme://user:pass@ahost:port/path", sts("random+scheme://user:pass@ahost:port/path"), ) self.assertEqual( "random+scheme://user:pass@ahost:port/path", sts("random+scheme://user:pass@ahost:port/path/"), ) self.assertEqual( "random+scheme://user:pass@ahost:port/", sts("random+scheme://user:pass@ahost:port/"), ) # Make sure relative paths work too self.assertEqual("path/to/foo", sts("path/to/foo")) self.assertEqual("path/to/foo", sts("path/to/foo/")) self.assertEqual("../to/foo", sts("../to/foo/")) self.assertEqual("path/../foo", sts("path/../foo/")) def test_unescape_for_display_utf8(self): # Test that URLs are converted to nice unicode strings for display def test(expected, url, encoding="utf-8"): disp_url = urlutils.unescape_for_display(url, encoding=encoding) self.assertIsInstance(disp_url, str) self.assertEqual(expected, disp_url) test("http://foo", "http://foo") if sys.platform == "win32": test("C:/foo/path", "file:///C|/foo/path") test("C:/foo/path", "file:///C:/foo/path") else: test("/foo/path", "file:///foo/path") test("http://foo/%2Fbaz", "http://foo/%2Fbaz") test("http://host/r\xe4ksm\xf6rg\xe5s", "http://host/r%C3%A4ksm%C3%B6rg%C3%A5s") # Make sure special escaped characters stay escaped test( "http://host/%3B%2F%3F%3A%40%26%3D%2B%24%2C%23", "http://host/%3B%2F%3F%3A%40%26%3D%2B%24%2C%23", ) # Can we handle sections that don't have utf-8 encoding? test( "http://host/%EE%EE%EE/r\xe4ksm\xf6rg\xe5s", "http://host/%EE%EE%EE/r%C3%A4ksm%C3%B6rg%C3%A5s", ) # Test encoding into output that can handle some characters test( "http://host/%EE%EE%EE/r\xe4ksm\xf6rg\xe5s", "http://host/%EE%EE%EE/r%C3%A4ksm%C3%B6rg%C3%A5s", encoding="iso-8859-1", ) # This one can be encoded into utf8 test( "http://host/\u062c\u0648\u062c\u0648", "http://host/%d8%ac%d9%88%d8%ac%d9%88", encoding="utf-8", ) # This can't be put into 8859-1 and so stays as escapes test( "http://host/%d8%ac%d9%88%d8%ac%d9%88", "http://host/%d8%ac%d9%88%d8%ac%d9%88", encoding="iso-8859-1", ) def test_escape(self): self.assertEqual("%25", urlutils.escape("%")) self.assertEqual("/~", urlutils.escape("/~")) self.assertEqual("/~", urlutils.escape("/~", safe="/")) self.assertEqual("%20", urlutils.escape(" ")) self.assertEqual("%C3%A5", urlutils.escape("\xe5")) self.assertEqual("%E5", urlutils.escape(b"\xe5")) self.assertIsInstance(urlutils.escape("\xe5"), str) def test_escape_tildes(self): self.assertEqual("~foo", urlutils.escape("~foo")) def test_unescape(self): self.assertEqual("%", urlutils.unescape("%25")) self.assertEqual("\xe5", urlutils.unescape("%C3%A5")) self.assertEqual("\xe5", urlutils.unescape("%C3%A5")) def test_escape_unescape(self): self.assertEqual("\xe5", urlutils.unescape(urlutils.escape("\xe5"))) self.assertEqual("%", urlutils.unescape(urlutils.escape("%"))) def test_relative_url(self): def test(expected, base, other): result = urlutils.relative_url(base, other) self.assertEqual(expected, result) test("a", "http://host/", "http://host/a") test( "http://entirely/different", "sftp://host/branch", "http://entirely/different", ) test( "../person/feature", "http://host/branch/mainline", "http://host/branch/person/feature", ) test("..", "http://host/branch", "http://host/") test("http://host2/branch", "http://host1/branch", "http://host2/branch") test(".", "http://host1/branch", "http://host1/branch") test( "../../../branch/2b", "file:///home/jelmer/foo/bar/2b", "file:///home/jelmer/branch/2b", ) test( "../../branch/2b", "sftp://host/home/jelmer/bar/2b", "sftp://host/home/jelmer/branch/2b", ) test( "../../branch/feature/%2b", "http://host/home/jelmer/bar/%2b", "http://host/home/jelmer/branch/feature/%2b", ) test( "../../branch/feature/2b", "http://host/home/jelmer/bar/2b/", "http://host/home/jelmer/branch/feature/2b", ) # relative_url should preserve a trailing slash test( "../../branch/feature/2b/", "http://host/home/jelmer/bar/2b/", "http://host/home/jelmer/branch/feature/2b/", ) test( "../../branch/feature/2b/", "http://host/home/jelmer/bar/2b", "http://host/home/jelmer/branch/feature/2b/", ) # TODO: treat http://host as http://host/ # relative_url is typically called from a branch.base or # transport.base which always ends with a / # test('a', 'http://host', 'http://host/a') test("http://host/a", "http://host", "http://host/a") # test('.', 'http://host', 'http://host/') test("http://host/", "http://host", "http://host/") # test('.', 'http://host/', 'http://host') test("http://host", "http://host/", "http://host") # On Windows file:///C:/path/to and file:///D:/other/path # should not use relative url over the non-existent '/' directory. if sys.platform == "win32": # on the same drive test("../../other/path", "file:///C:/path/to", "file:///C:/other/path") # ~next two tests is failed, i.e. urlutils.relative_url expects # ~to see normalized file URLs? # ~test('../../other/path', # ~ 'file:///C:/path/to', 'file:///c:/other/path') # ~test('../../other/path', # ~ 'file:///C:/path/to', 'file:///C|/other/path') # check UNC paths too test( "../../other/path", "file://HOST/base/path/to", "file://HOST/base/other/path", ) # on different drives test("file:///D:/other/path", "file:///C:/path/to", "file:///D:/other/path") # TODO: strictly saying in UNC path //HOST/base is full analog # of drive letter for hard disk, and this situation is also # should be exception from rules. [bialix 20071221] class TestCwdToURL(TestCaseInTempDir): """Test that local_path_to_url works based on the cwd.""" def test_dot(self): # This test will fail if getcwd is not ascii os.mkdir("mytest") os.chdir("mytest") url = urlutils.local_path_to_url(".") self.assertEndsWith(url, "/mytest") def test_non_ascii(self): try: os.mkdir("dod\xe9") except UnicodeError as err: raise TestSkipped("cannot create unicode directory") from err os.chdir("dod\xe9") # On Mac OSX this directory is actually: # u'/dode\u0301' => '/dode\xcc\x81 # but we should normalize it back to # u'/dod\xe9' => '/dod\xc3\xa9' url = urlutils.local_path_to_url(".") self.assertEndsWith(url, "/dod%C3%A9") class TestDeriveToLocation(TestCase): """Test that the mapping of FROM_LOCATION to TO_LOCATION works.""" def test_to_locations_derived_from_paths(self): derive = urlutils.derive_to_location self.assertEqual("bar", derive("bar")) self.assertEqual("bar", derive("../bar")) self.assertEqual("bar", derive("/foo/bar")) self.assertEqual("bar", derive("c:/foo/bar")) self.assertEqual("bar", derive("c:bar")) def test_to_locations_derived_from_urls(self): derive = urlutils.derive_to_location self.assertEqual("bar", derive("http://foo/bar")) self.assertEqual("bar", derive("bzr+ssh://foo/bar")) self.assertEqual("foo-bar", derive("lp:foo-bar")) class TestRebaseURL(TestCase): """Test the behavior of rebase_url.""" def test_non_relative(self): result = urlutils.rebase_url("file://foo", "file://foo", "file://foo/bar") self.assertEqual("file://foo", result) result = urlutils.rebase_url("/foo", "file://foo", "file://foo/bar") self.assertEqual("/foo", result) def test_different_ports(self): e = self.assertRaises( urlutils.InvalidRebaseURLs, urlutils.rebase_url, "foo", "http://bar:80", "http://bar:81", ) self.assertEqual( str(e), "URLs differ by more than path: 'http://bar:80' and 'http://bar:81'", ) def test_different_hosts(self): e = self.assertRaises( urlutils.InvalidRebaseURLs, urlutils.rebase_url, "foo", "http://bar", "http://baz", ) self.assertEqual( str(e), "URLs differ by more than path: 'http://bar' and 'http://baz'" ) def test_different_protocol(self): e = self.assertRaises( urlutils.InvalidRebaseURLs, urlutils.rebase_url, "foo", "http://bar", "ftp://bar", ) self.assertEqual( str(e), "URLs differ by more than path: 'http://bar' and 'ftp://bar'" ) def test_rebase_success(self): self.assertEqual( "../bar", urlutils.rebase_url("bar", "http://baz/", "http://baz/qux") ) self.assertEqual( "qux/bar", urlutils.rebase_url("bar", "http://baz/qux", "http://baz/") ) self.assertEqual( ".", urlutils.rebase_url("foo", "http://bar/", "http://bar/foo/") ) self.assertEqual( "qux/bar", urlutils.rebase_url("../bar", "http://baz/qux/foo", "http://baz/"), ) def test_determine_relative_path(self): self.assertEqual( "../../baz/bar", urlutils.determine_relative_path("/qux/quxx", "/baz/bar") ) self.assertEqual("..", urlutils.determine_relative_path("/bar/baz", "/bar")) self.assertEqual("baz", urlutils.determine_relative_path("/bar", "/bar/baz")) self.assertEqual(".", urlutils.determine_relative_path("/bar", "/bar")) class TestParseURL(TestCase): def test_parse_simple(self): parsed = urlutils.parse_url("http://example.com:80/one") self.assertEqual(("http", None, None, "example.com", 80, "/one"), parsed) def test_ipv6(self): parsed = urlutils.parse_url("http://[1:2:3::40]/one") self.assertEqual(("http", None, None, "1:2:3::40", None, "/one"), parsed) def test_ipv6_port(self): parsed = urlutils.parse_url("http://[1:2:3::40]:80/one") self.assertEqual(("http", None, None, "1:2:3::40", 80, "/one"), parsed) class TestURL(TestCase): def test_parse_simple(self): parsed = urlutils.URL.from_string("http://example.com:80/one") self.assertEqual("http", parsed.scheme) self.assertIs(None, parsed.user) self.assertIs(None, parsed.password) self.assertEqual("example.com", parsed.host) self.assertEqual(80, parsed.port) self.assertEqual("/one", parsed.path) def test_ipv6(self): parsed = urlutils.URL.from_string("http://[1:2:3::40]/one") self.assertEqual("http", parsed.scheme) self.assertIs(None, parsed.port) self.assertIs(None, parsed.user) self.assertIs(None, parsed.password) self.assertEqual("1:2:3::40", parsed.host) self.assertEqual("/one", parsed.path) def test_ipv6_port(self): parsed = urlutils.URL.from_string("http://[1:2:3::40]:80/one") self.assertEqual("http", parsed.scheme) self.assertEqual("1:2:3::40", parsed.host) self.assertIs(None, parsed.user) self.assertIs(None, parsed.password) self.assertEqual(80, parsed.port) self.assertEqual("/one", parsed.path) def test_quoted(self): parsed = urlutils.URL.from_string( "http://ro%62ey:h%40t@ex%41mple.com:2222/path" ) self.assertEqual(parsed.quoted_host, "ex%41mple.com") self.assertEqual(parsed.host, "exAmple.com") self.assertEqual(parsed.port, 2222) self.assertEqual(parsed.quoted_user, "ro%62ey") self.assertEqual(parsed.user, "robey") self.assertEqual(parsed.quoted_password, "h%40t") self.assertEqual(parsed.password, "h@t") self.assertEqual(parsed.path, "/path") def test_eq(self): parsed1 = urlutils.URL.from_string("http://[1:2:3::40]:80/one") parsed2 = urlutils.URL.from_string("http://[1:2:3::40]:80/one") self.assertEqual(parsed1, parsed2) self.assertEqual(parsed1, parsed1) parsed2.path = "/two" self.assertNotEqual(parsed1, parsed2) def test_repr(self): parsed = urlutils.URL.from_string("http://[1:2:3::40]:80/one") self.assertEqual( "", repr(parsed) ) def test_str(self): parsed = urlutils.URL.from_string("http://[1:2:3::40]:80/one") self.assertEqual("http://[1:2:3::40]:80/one", str(parsed)) def test_combine_paths(self): combine = urlutils.combine_paths self.assertEqual( "/home/sarah/project/foo", combine("/home/sarah", "project/foo") ) self.assertEqual("/etc", combine("/home/sarah", "../../etc")) self.assertEqual("/etc", combine("/home/sarah", "../../../etc")) self.assertEqual("/etc", combine("/home/sarah", "/etc")) def test_clone(self): url = urlutils.URL.from_string("http://[1:2:3::40]:80/one") url1 = url.clone("two") self.assertEqual("/one/two", url1.path) url2 = url.clone("/two") self.assertEqual("/two", url2.path) url3 = url.clone() self.assertIsNot(url, url3) self.assertEqual(url, url3) def test_parse_empty_port(self): parsed = urlutils.URL.from_string("http://example.com:/one") self.assertEqual("http", parsed.scheme) self.assertIs(None, parsed.user) self.assertIs(None, parsed.password) self.assertEqual("example.com", parsed.host) self.assertIs(None, parsed.port) self.assertEqual("/one", parsed.path) class TestFileRelpath(TestCase): # GZ 2011-11-18: A way to override all path handling functions to one # platform or another for testing would be nice. def _with_posix_paths(self): self.overrideAttr( urlutils, "local_path_from_url", urlutils._posix_local_path_from_url ) self.overrideAttr(urlutils, "MIN_ABS_FILEURL_LENGTH", len("file:///")) self.overrideAttr(osutils, "normpath", osutils._posix_normpath) self.overrideAttr(osutils, "abspath", posixpath.abspath) self.overrideAttr(osutils, "normpath", osutils._posix_normpath) self.overrideAttr(osutils, "pathjoin", posixpath.join) self.overrideAttr(osutils, "split", posixpath.split) self.overrideAttr(osutils, "MIN_ABS_PATHLENGTH", 1) def _with_win32_paths(self): self.overrideAttr( urlutils, "local_path_from_url", urlutils._win32_local_path_from_url ) self.overrideAttr( urlutils, "MIN_ABS_FILEURL_LENGTH", urlutils.WIN32_MIN_ABS_FILEURL_LENGTH ) self.overrideAttr(osutils, "abspath", osutils._win32_abspath) self.overrideAttr(osutils, "normpath", osutils._win32_normpath) self.overrideAttr(osutils, "split", ntpath.split) self.overrideAttr(osutils, "MIN_ABS_PATHLENGTH", 3) def test_same_url_posix(self): self._with_posix_paths() self.assertEqual("", urlutils.file_relpath("file:///a", "file:///a")) self.assertEqual("", urlutils.file_relpath("file:///a", "file:///a/")) self.assertEqual("", urlutils.file_relpath("file:///a/", "file:///a")) def test_same_url_win32(self): if sys.platform != "win32": raise TestSkipped( "broken on non-windows; _with_win32_paths no longer works for rust" ) self._with_win32_paths() self.assertEqual("", urlutils.file_relpath("file:///A:/", "file:///A:/")) self.assertEqual("", urlutils.file_relpath("file:///A|/", "file:///A:/")) self.assertEqual("", urlutils.file_relpath("file:///A:/b/", "file:///A:/b/")) self.assertEqual("", urlutils.file_relpath("file:///A:/b", "file:///A:/b/")) self.assertEqual("", urlutils.file_relpath("file:///A:/b/", "file:///A:/b")) def test_child_posix(self): self._with_posix_paths() self.assertEqual("b", urlutils.file_relpath("file:///a", "file:///a/b")) self.assertEqual("b", urlutils.file_relpath("file:///a/", "file:///a/b")) self.assertEqual("b/c", urlutils.file_relpath("file:///a", "file:///a/b/c")) def test_child_win32(self): if sys.platform != "win32": raise TestSkipped( "broken on non-windows; _with_win32_paths no longer works for rust" ) self._with_win32_paths() self.assertEqual("b", urlutils.file_relpath("file:///A:/", "file:///A:/b")) self.assertEqual("b", urlutils.file_relpath("file:///A|/", "file:///A:/b")) self.assertEqual("c", urlutils.file_relpath("file:///A:/b", "file:///A:/b/c")) self.assertEqual("c", urlutils.file_relpath("file:///A:/b/", "file:///A:/b/c")) self.assertEqual( "c/d", urlutils.file_relpath("file:///A:/b", "file:///A:/b/c/d") ) def test_sibling_posix(self): self._with_posix_paths() self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///a/b", "file:///a/c" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///a/b/", "file:///a/c" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///a/b/", "file:///a/c/" ) def test_sibling_win32(self): self._with_win32_paths() self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///A:/b", "file:///A:/c" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///A:/b/", "file:///A:/c" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///A:/b/", "file:///A:/c/" ) def test_parent_posix(self): self._with_posix_paths() self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///a/b", "file:///a" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///a/b", "file:///a/" ) def test_parent_win32(self): self._with_win32_paths() self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///A:/b", "file:///A:/" ) self.assertRaises( PathNotChild, urlutils.file_relpath, "file:///A:/b/c", "file:///A:/b" ) class QuoteTests(TestCase): def test_quote(self): self.assertEqual("abc%20def", urlutils.quote("abc def")) self.assertEqual("abc%2Fdef", urlutils.quote("abc/def", safe="")) self.assertEqual("abc/def", urlutils.quote("abc/def", safe="/")) def test_quote_tildes(self): self.assertEqual("~foo", urlutils.quote("~foo")) self.assertEqual("~foo", urlutils.quote("~foo", safe="/~")) def test_unquote(self): self.assertEqual("%", urlutils.unquote("%25")) self.assertEqual("\xe5", urlutils.unquote("%C3%A5")) self.assertEqual("\xe5", urlutils.unquote("\xe5")) def test_unquote_to_bytes(self): self.assertEqual(b"%", urlutils.unquote_to_bytes("%25")) self.assertEqual(b"\xc3\xa5", urlutils.unquote_to_bytes("%C3%A5")) dromedary-0.1.1/tests/transport_util.py000064400000000000000000000031321046102023000164150ustar 00000000000000# Copyright (C) 2007-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Utilities for testing transport connections and hooks.""" from dromedary import Transport # SFTPTransport is the only bundled transport that properly counts connections # at the moment. from . import test_sftp_transport class TestCaseWithConnectionHookedTransport(test_sftp_transport.TestCaseWithSFTPServer): """Test case that tracks transport connections using hooks.""" def setUp(self): """Set up the test case with connection tracking.""" super().setUp() self.reset_connections() def start_logging_connections(self): """Start logging transport connections using transport hooks.""" Transport.hooks.install_named_hook( "post_connect", self.connections.append, None ) def reset_connections(self): """Reset the connections list to start fresh tracking.""" self.connections = [] dromedary-0.1.1/trace.py000064400000000000000000000161731046102023000132710ustar 00000000000000# Copyright (C) 2007 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Implementation of Transport that traces transport operations. This does not change the transport behaviour at all, merely records every call and then delegates it. """ from dromedary import decorator class TransportTraceDecorator(decorator.TransportDecorator): """A tracing decorator for Transports. Calls that potentially perform IO are logged to self._activity. The _activity attribute is shared as the transport is cloned, but not if a new transport is created without cloning. Not all operations are logged at this point, if you need an unlogged operation please add a test to the tests of this transport, for the logging of the operation you want logged. See also TransportLogDecorator, that records a machine-readable log in memory for eg testing. """ def __init__(self, url, _decorated=None, _from_transport=None): """Set the 'base' path where files will be stored. _decorated is a private parameter for cloning. """ super().__init__(url, _decorated) if _from_transport is None: # newly created self._activity = [] else: # cloned self._activity = _from_transport._activity def append_file(self, relpath, f, mode=None): """See Transport.append_file().""" return self._decorated.append_file(relpath, f, mode=mode) def append_bytes(self, relpath, bytes, mode=None): """See Transport.append_bytes().""" return self._decorated.append_bytes(relpath, bytes, mode=mode) def delete(self, relpath): """See Transport.delete().""" self._activity.append(("delete", relpath)) return self._decorated.delete(relpath) def delete_tree(self, relpath): """See Transport.delete_tree().""" return self._decorated.delete_tree(relpath) @classmethod def _get_url_prefix(self): """Tracing transports are identified by 'trace+'.""" return "trace+" def get(self, relpath): """See Transport.get().""" self._trace(("get", relpath)) return self._decorated.get(relpath) def get_smart_client(self): """Get a smart protocol client from the decorated transport. Returns: The smart client from the underlying transport. """ return self._decorated.get_smart_client() def has(self, relpath): """See Transport.has().""" return self._decorated.has(relpath) def is_readonly(self): """See Transport.is_readonly.""" return self._decorated.is_readonly() def mkdir(self, relpath, mode=None): """See Transport.mkdir().""" self._trace(("mkdir", relpath, mode)) return self._decorated.mkdir(relpath, mode) def open_write_stream(self, relpath, mode=None): """See Transport.open_write_stream.""" return self._decorated.open_write_stream(relpath, mode=mode) def put_file(self, relpath, f, mode=None): """See Transport.put_file().""" return self._decorated.put_file(relpath, f, mode) def put_bytes(self, relpath: str, raw_bytes: bytes, mode=None): """See Transport.put_bytes().""" self._trace(("put_bytes", relpath, len(raw_bytes), mode)) return self._decorated.put_bytes(relpath, raw_bytes, mode) def put_bytes_non_atomic( self, relpath: str, raw_bytes: bytes, mode=None, create_parent_dir=False, dir_mode=None, ): """See Transport.put_bytes_non_atomic.""" self._trace( ( "put_bytes_non_atomic", relpath, len(raw_bytes), mode, create_parent_dir, dir_mode, ) ) return self._decorated.put_bytes_non_atomic( relpath, raw_bytes, mode=mode, create_parent_dir=create_parent_dir, dir_mode=dir_mode, ) def listable(self): """See Transport.listable.""" return self._decorated.listable() def iter_files_recursive(self): """See Transport.iter_files_recursive().""" return self._decorated.iter_files_recursive() def list_dir(self, relpath): """See Transport.list_dir().""" return self._decorated.list_dir(relpath) def readv(self, relpath, offsets, adjust_for_latency=False, upper_limit=None): """Read multiple ranges from a file. Args: relpath: Path relative to transport root. offsets: List of (offset, length) tuples to read. adjust_for_latency: Whether to adjust reading for latency. upper_limit: Maximum amount to read. Returns: Iterator of (offset, data) tuples. """ # we override at the readv() level rather than _readv() so that any # latency adjustments will be done by the underlying transport self._trace(("readv", relpath, offsets, adjust_for_latency, upper_limit)) return self._decorated.readv(relpath, offsets, adjust_for_latency, upper_limit) def recommended_page_size(self): """See Transport.recommended_page_size().""" return self._decorated.recommended_page_size() def rename(self, rel_from, rel_to): """Rename a file or directory. Args: rel_from: Current relative path. rel_to: New relative path. """ self._activity.append(("rename", rel_from, rel_to)) return self._decorated.rename(rel_from, rel_to) def rmdir(self, relpath): """See Transport.rmdir.""" self._trace(("rmdir", relpath)) return self._decorated.rmdir(relpath) def stat(self, relpath): """See Transport.stat().""" return self._decorated.stat(relpath) def lock_read(self, relpath): """See Transport.lock_read.""" return self._decorated.lock_read(relpath) def lock_write(self, relpath): """See Transport.lock_write.""" return self._decorated.lock_write(relpath) def _trace(self, operation_tuple): """Record that a transport operation occured. :param operation: Tuple of transport call name and arguments. """ self._activity.append(operation_tuple) def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [(TransportTraceDecorator, test_server.TraceServer)] dromedary-0.1.1/unlistable.py000064400000000000000000000022521046102023000143260ustar 00000000000000# Copyright (C) 2005, 2006 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Transport implementation that disables listing to simulate HTTP cheaply.""" from dromedary._transport_rs.unlistable import UnlistableTransportDecorator __all__ = ["UnlistableTransportDecorator", "get_test_permutations"] def get_test_permutations(): """Return the permutations to be used in testing.""" from dromedary.tests import test_server return [ (UnlistableTransportDecorator, test_server.UnlistableServer), ] dromedary-0.1.1/urlutils.py000064400000000000000000000177111046102023000140550ustar 00000000000000# Copyright (C) 2006-2010 Canonical Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """A collection of function for handling URL operations.""" import sys from urllib import parse as urlparse from dromedary.osutils import pathjoin, splitpath from . import errors class InvalidURL(errors.PathError): """Exception raised when an invalid URL is encountered. This is raised when a URL string cannot be parsed or is malformed. """ _fmt = "Invalid url supplied to transport: %(path)r%(extra)s" class InvalidURLJoin(errors.PathError): """Exception raised when a URL join operation is invalid.""" _fmt = "Invalid URL join request: %(reason)s: %(base)r + %(join_args)r" def __init__(self, reason, base, join_args): """Initialize with the join failure reason, base URL, and join args.""" self.reason = reason self.base = base self.join_args = join_args errors.PathError.__init__(self, base, reason) class InvalidRebaseURLs(errors.TransportError): """Exception raised when URLs cannot be rebased. This occurs when trying to rebase URLs that differ by more than just their paths. """ _fmt = "URLs differ by more than path: %(old_base)r and %(new_base)r" def __init__(self, old_base, new_base): """Initialize with the original and new base URLs.""" self.old_base = old_base self.new_base = new_base errors.TransportError.__init__(self) quote_from_bytes = urlparse.quote_from_bytes quote = urlparse.quote unquote_to_bytes = urlparse.unquote_to_bytes unquote = urlparse.unquote from ._transport_rs.urlutils import ( # noqa: F401 URL, _find_scheme_and_separator, basename, combine_paths, derive_to_location, dirname, escape, file_relpath, is_url, join, join_segment_parameters, join_segment_parameters_raw, joinpath, local_path_from_url, local_path_to_url, normalize_url, parse_url, relative_url, split, split_segment_parameters, split_segment_parameters_raw, strip_segment_parameters, strip_trailing_slash, unescape, ) from ._transport_rs.urlutils import posix as posix_rs from ._transport_rs.urlutils import win32 as win32_rs _posix_local_path_to_url = posix_rs.local_path_to_url _win32_local_path_to_url = win32_rs.local_path_to_url _win32_local_path_from_url = win32_rs.local_path_from_url _posix_local_path_from_url = posix_rs.local_path_from_url MIN_ABS_FILEURL_LENGTH = len("file:///") WIN32_MIN_ABS_FILEURL_LENGTH = len("file:///C:/") if sys.platform == "win32": MIN_ABS_FILEURL_LENGTH = WIN32_MIN_ABS_FILEURL_LENGTH _win32_extract_drive_letter = win32_rs.extract_drive_letter _win32_strip_local_trailing_slash = win32_rs.strip_local_trailing_slash # These are characters that if escaped, should stay that way _no_decode_chars = ";/?:@&=+$,#" _no_decode_ords = [ord(c) for c in _no_decode_chars] _no_decode_hex = [f"{o:02x}" for o in _no_decode_ords] + [ f"{o:02X}" for o in _no_decode_ords ] _hex_display_map = dict( [(f"{o:02x}", bytes([o])) for o in range(256)] + [(f"{o:02X}", bytes([o])) for o in range(256)] ) # These entries get mapped to themselves _hex_display_map.update((hex, b"%" + hex.encode("ascii")) for hex in _no_decode_hex) # These characters shouldn't be percent-encoded, and it's always safe to # unencode them if they are. _url_dont_escape_characters = set( "abcdefghijklmnopqrstuvwxyz" # Lowercase alpha "ABCDEFGHIJKLMNOPQRSTUVWXYZ" # Uppercase alpha "0123456789" # Numbers "-._~" # Unreserved characters ) def _unescape_segment_for_display(segment, encoding): """Unescape a segment for display. Helper for unescape_for_display Args: url: A 7-bit ASCII URL encoding: The final output encoding Returns: A unicode string which can be safely encoded into the specified encoding. """ escaped_chunks = segment.split("%") escaped_chunks[0] = escaped_chunks[0].encode("utf-8") for j in range(1, len(escaped_chunks)): item = escaped_chunks[j] try: escaped_chunks[j] = _hex_display_map[item[:2]] except KeyError: # Put back the percent symbol escaped_chunks[j] = b"%" + (item[:2].encode("utf-8")) except UnicodeDecodeError: escaped_chunks[j] = chr(int(item[:2], 16)).encode("utf-8") escaped_chunks[j] += item[2:].encode("utf-8") unescaped = b"".join(escaped_chunks) try: decoded = unescaped.decode("utf-8") except UnicodeDecodeError: # If this path segment cannot be properly utf-8 decoded # after doing unescaping we will just leave it alone return segment else: try: decoded.encode(encoding) except UnicodeEncodeError: # If this chunk cannot be encoded in the local # encoding, then we should leave it alone return segment else: # Otherwise take the url decoded one return decoded def unescape_for_display(url, encoding): """Decode what you can for a URL, so that we get a nice looking path. This will turn file:// urls into local paths, and try to decode any portions of a http:// style url that it can. Any sections of the URL which can't be represented in the encoding or need to stay as escapes are left alone. Args: url: A 7-bit ASCII URL encoding: The final output encoding Returns: A unicode string which can be safely encoded into the specified encoding. """ if encoding is None: raise ValueError("you cannot specify None for the display encoding") if url.startswith("file://"): try: path = local_path_from_url(url) str(path).encode(encoding) return path except UnicodeError: return url # Split into sections to try to decode utf-8 res = url.split("/") for i in range(1, len(res)): res[i] = _unescape_segment_for_display(res[i], encoding) return "/".join(res) def _is_absolute(url): return url.startswith("/") def rebase_url(url, old_base, new_base): """Convert a relative path from an old base URL to a new base URL. The result will be a relative path. Absolute paths and full URLs are returned unaltered. """ scheme, _separator = _find_scheme_and_separator(url) if scheme is not None: return url if _is_absolute(url): return url old_parsed = urlparse.urlparse(old_base) new_parsed = urlparse.urlparse(new_base) if (old_parsed[:2]) != (new_parsed[:2]): raise InvalidRebaseURLs(old_base, new_base) return determine_relative_path(new_parsed[2], join(old_parsed[2], url)) def determine_relative_path(from_path, to_path): """Determine a relative path from from_path to to_path.""" from_segments = splitpath(from_path) to_segments = splitpath(to_path) count = -1 for count, (from_element, to_element) in enumerate( # noqa: B007 zip( from_segments, to_segments, strict=False, ) ): if from_element != to_element: break else: count += 1 unique_from = from_segments[count:] unique_to = to_segments[count:] segments = [".."] * len(unique_from) + unique_to if len(segments) == 0: return "." return pathjoin(*segments) dromedary-0.1.1/version.py000064400000000000000000000040311046102023000136460ustar 00000000000000#!/usr/bin/env python3 # Copyright (C) 2024 Jelmer Vernooij # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA """Version information for dromedary.""" __version__ = "0.1.0" # Version information for dromedary version_info = (0, 1, 0, "final", 0) def _format_version_tuple(version_info): """Format version tuple into a version string. Args: version_info: Tuple of (major, minor, micro, release_type, sub) Returns: Formatted version string """ if len(version_info) == 2: main_version = "%d.%d" % version_info[:2] else: main_version = "%d.%d.%d" % version_info[:3] if len(version_info) <= 3: return main_version release_type = version_info[3] sub = version_info[4] if release_type == "final" and sub == 0: sub_string = "" elif release_type == "final": sub_string = "." + str(sub) elif release_type == "dev" and sub == 0: sub_string = ".dev" elif release_type == "dev": sub_string = ".dev" + str(sub) elif release_type in ("alpha", "beta"): if version_info[2] == 0: main_version = "%d.%d" % version_info[:2] sub_string = "." + release_type[0] + str(sub) elif release_type == "candidate": sub_string = ".rc" + str(sub) else: return ".".join(map(str, version_info)) return main_version + sub_string version_string = __version__