diff --git a/Cargo.lock b/Cargo.lock index 89df972..e6384af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.17.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ "gimli", ] @@ -18,14 +18,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "atty" -version = "0.2.14" +name = "aes" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "hermit-abi", - "libc", - "winapi", + "cfg-if", + "cipher", + "cpufeatures", + "opaque-debug", ] [[package]] @@ -36,9 +37,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", @@ -49,6 +50,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64ct" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" + [[package]] name = "bitflags" version = "1.3.2" @@ -56,13 +63,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "bstr" -version = "0.2.17" +name = "block-buffer" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bstr" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b" dependencies = [ - "lazy_static", "memchr", + "once_cell", "regex-automata", "serde", ] @@ -81,15 +97,39 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] [[package]] name = "cc" -version = "1.0.74" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -98,15 +138,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "clap" -version = "4.0.18" +name = "cipher" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + +[[package]] +name = "clap" +version = "4.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" dependencies = [ - "atty", "bitflags", "clap_derive", "clap_lex", + "is-terminal", "once_cell", "strsim", "termcolor", @@ -116,9 +165,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.0.18" +version = "4.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3" +checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ "heck", "proc-macro-error", @@ -176,10 +225,53 @@ dependencies = [ ] [[package]] -name = "csv-async" -version = "1.2.4" +name = "constant_time_eq" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19b33b32fd48f83388821bd8f534b59e1b1ffd5c6c83771d1b23abd3dac2685" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv-async" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c652d4c48e4dc80b26fadd169c02fb6053d9f57507ddd3e6b8706e7d0242235e" dependencies = [ "bstr", "cfg-if", @@ -201,6 +293,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "directories" version = "4.0.1" @@ -241,9 +344,32 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", + "toml", "tracing", "tracing-error", "tracing-subscriber", + "zip", +] + +[[package]] +name = "errno" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +dependencies = [ + "errno-dragonfly", + "libc", + "winapi", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", ] [[package]] @@ -265,6 +391,16 @@ dependencies = [ "instant", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "futures" version = "0.3.25" @@ -354,6 +490,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.8" @@ -367,15 +513,15 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.2" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +checksum = "dec7af912d60cdbd3677c1af9352ebae6fb8394d165568a2234df0fa00f87793" [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "heck" @@ -385,13 +531,22 @@ checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" [[package]] name = "hermit-abi" -version = "0.1.19" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "indenter" version = "0.3.3" @@ -408,10 +563,41 @@ dependencies = [ ] [[package]] -name = "itoa" -version = "1.0.4" +name = "io-lifetimes" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +dependencies = [ + "hermit-abi", + "io-lifetimes", + "rustix", + "windows-sys", +] + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] [[package]] name = "lazy_static" @@ -421,9 +607,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libloading" @@ -435,6 +621,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "log" version = "0.4.17" @@ -467,9 +659,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -494,9 +686,9 @@ checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" [[package]] name = "nom" -version = "7.1.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c" dependencies = [ "memchr", "minimal-lexical", @@ -525,9 +717,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ "hermit-abi", "libc", @@ -535,24 +727,30 @@ dependencies = [ [[package]] name = "object" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "8d864c91689fdc196779b98dba0aceac6118594c2df6ee5d943eb6a8df4d107a" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "os_str_bytes" -version = "6.3.1" +version = "6.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" [[package]] name = "overload" @@ -566,6 +764,29 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest", + "hmac", + "password-hash", + "sha2", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -578,6 +799,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -604,22 +831,28 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -642,9 +875,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "regex-syntax", ] @@ -660,9 +893,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "remove_dir_all" @@ -680,10 +913,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] -name = "ryu" -version = "1.0.11" +name = "rustix" +version = "0.36.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "sdk" @@ -708,18 +955,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.147" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -735,6 +982,28 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -775,10 +1044,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "syn" -version = "1.0.103" +name = "subtle" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -810,18 +1085,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -838,10 +1113,37 @@ dependencies = [ ] [[package]] -name = "tokio" -version = "1.21.2" +name = "time" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +dependencies = [ + "time-core", +] + +[[package]] +name = "tokio" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -853,14 +1155,14 @@ dependencies = [ "signal-hook-registry", "tokio-macros", "tracing", - "winapi", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.8.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", @@ -880,9 +1182,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] @@ -959,6 +1261,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + [[package]] name = "unicase" version = "2.6.0" @@ -970,9 +1278,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-width" @@ -1085,3 +1393,52 @@ name = "windows_x86_64_msvc" version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + +[[package]] +name = "zip" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537ce7411d25e54e8ae21a7ce0b15840e7bfcff15b51d697ec3266cc76bdf080" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.4+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" +dependencies = [ + "cc", + "libc", +] diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 2aa46ac..0181b59 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -22,6 +22,8 @@ tracing = { version = "0.1.37", features = ["async-await"] } tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } confy = "0.5.1" +toml = "0.5.9" +zip = "0.6.3" [dev-dependencies] tempfile = "3.3.0" diff --git a/crates/dtmt/src/cmd/build.rs b/crates/dtmt/src/cmd/build.rs index 0a02406..877b93d 100644 --- a/crates/dtmt/src/cmd/build.rs +++ b/crates/dtmt/src/cmd/build.rs @@ -1,22 +1,229 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::Result; +use color_eyre::eyre::{self, Context, Result}; +use color_eyre::{Help, Report}; +use futures::future::try_join_all; +use futures::StreamExt; +use sdk::filetype::package::Package; +use sdk::{Bundle, BundleFile, Oodle}; +use serde::Deserialize; +use tokio::fs::{self, File}; +use tokio::io::AsyncReadExt; -pub(crate) fn _command_definition() -> Command { - Command::new("build").about("Build a project").arg( - Arg::new("directory") - .required(false) - .default_value(".") - .value_parser(value_parser!(PathBuf)) - .help( - "The path to the project to build. \ - If omitted, the current working directory is used.", - ), - ) +use crate::mods::archive::Archive; + +pub(crate) fn command_definition() -> Command { + Command::new("build") + .about("Build a project") + .arg( + Arg::new("directory") + .required(false) + .value_parser(value_parser!(PathBuf)) + .help( + "The path to the project to build. \ + If omitted, dtmt will search from the current working directory upward.", + ), + ) + .arg( + Arg::new("oodle") + .long("oodle") + .default_value(super::OODLE_LIB_NAME) + .help( + "The oodle library to load. This may either be:\n\ + - A library name that will be searched for in the system's default paths.\n\ + - A file path relative to the current working directory.\n\ + - An absolute file path.", + ), + ) +} + +#[derive(Debug, Default, Deserialize)] +struct ProjectConfig { + #[serde(skip)] + dir: PathBuf, + name: String, + packages: Vec, +} + +#[tracing::instrument] +async fn find_project_config(dir: Option) -> Result { + let (path, mut file) = if let Some(path) = dir { + let file = File::open(&path.join("dtmt.toml")) + .await + .wrap_err_with(|| format!("failed to open file: {}", path.display())) + .with_suggestion(|| { + format!( + "Make sure the file at '{}' exists and is readable", + path.display() + ) + })?; + (path, file) + } else { + let mut dir = std::env::current_dir()?; + loop { + let path = dir.join("dtmt.toml"); + match File::open(&path).await { + Ok(file) => break (path, file), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + if let Some(parent) = dir.parent() { + // TODO: Re-write with recursion to avoid allocating the `PathBuf`. + dir = parent.to_path_buf(); + } else { + eyre::bail!("Could not find project root"); + } + } + Err(err) => { + let err = Report::new(err) + .wrap_err(format!("failed to open file: {}", path.display())); + return Err(err); + } + } + } + }; + + let mut buf = Vec::new(); + file.read_to_end(&mut buf).await?; + + let mut cfg: ProjectConfig = toml::from_slice(&buf)?; + cfg.dir = path; + Ok(cfg) } #[tracing::instrument(skip_all)] -pub(crate) async fn run(_ctx: sdk::Context, _matches: &ArgMatches) -> Result<()> { - unimplemented!() +async fn compile_package_files

(pkg: &Package, root: P) -> Result> +where + P: AsRef + std::fmt::Debug, +{ + let root = Arc::new(root.as_ref()); + + let tasks = pkg + .iter() + .flat_map(|(file_type, paths)| { + paths.iter().map(|path| { + ( + *file_type, + path, + // Cloning the `Arc` here solves the issue that in the next `.map`, I need to + // `move` the closure parameters, but can't `move` `root` before it was cloned. + root.clone(), + ) + }) + }) + .map(|(file_type, path, root)| async move { + let sjson = fs::read_to_string(&path).await?; + BundleFile::from_sjson(file_type, sjson, root.as_ref()).await + }); + + let results = futures::stream::iter(tasks) + .buffer_unordered(10) + .collect::>>() + .await; + + results.into_iter().collect() +} + +#[tracing::instrument(skip_all, fields(files = files.len()))] +fn compile_bundle(name: String, files: Vec) -> Result { + let mut bundle = Bundle::new(name); + + for file in files { + bundle.add_file(file); + } + + Ok(bundle) +} + +#[tracing::instrument] +async fn build_package(package: P1, root: P2) -> Result +where + P1: AsRef + std::fmt::Debug, + P2: AsRef + std::fmt::Debug, +{ + let root = root.as_ref(); + let package = package.as_ref(); + + let mut path = root.join(package); + path.set_extension("package"); + let sjson = fs::read_to_string(&path) + .await + .wrap_err_with(|| format!("failed to read file {}", path.display()))?; + + let pkg_name = package.to_string_lossy().to_string(); + let pkg = Package::from_sjson(sjson, pkg_name.clone(), root) + .await + .wrap_err_with(|| format!("invalid package file {}", &pkg_name))?; + + compile_package_files(&pkg, root) + .await + .wrap_err("failed to compile package") + .and_then(|files| compile_bundle(pkg_name, files)) + .wrap_err("failed to build bundle") +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + if let Some(name) = matches.get_one::("oodle") { + let oodle = Oodle::new(name)?; + ctx.oodle = Some(oodle); + } + + let cfg = { + let dir = matches.get_one::("directory").cloned(); + find_project_config(dir).await? + }; + + let dest = { + let mut path = PathBuf::from(&cfg.name); + path.set_extension("zip"); + Arc::new(path) + }; + let cfg = Arc::new(cfg); + + tracing::debug!(?cfg); + + let tasks = cfg + .packages + .iter() + .map(|path| (path, cfg.clone())) + .map(|(path, cfg)| async move { + build_package(path, &cfg.dir).await.wrap_err_with(|| { + format!( + "failed to build package {} in {}", + path.display(), + cfg.dir.display() + ) + }) + }); + + let bundles = try_join_all(tasks).await?; + + let mod_file = { + let mut path = cfg.dir.join(&cfg.name); + path.set_extension("mod"); + fs::read(path).await? + }; + + { + let dest = dest.clone(); + let name = cfg.name.clone(); + tokio::task::spawn_blocking(move || { + let mut archive = Archive::new(name); + + archive.add_mod_file(mod_file); + + for bundle in bundles { + archive.add_bundle(bundle); + } + + archive + .write(&ctx, dest.as_ref()) + .wrap_err("failed to write mod archive") + }) + .await??; + } + + tracing::info!("Mod archive written to {}", dest.display()); + Ok(()) } diff --git a/crates/dtmt/src/cmd/bundle/mod.rs b/crates/dtmt/src/cmd/bundle/mod.rs index b287bcb..17f7a8c 100644 --- a/crates/dtmt/src/cmd/bundle/mod.rs +++ b/crates/dtmt/src/cmd/bundle/mod.rs @@ -7,12 +7,6 @@ mod extract; mod inject; mod list; -#[cfg(target_os = "windows")] -const OODLE_LIB_NAME: &str = "oo2core_8_win64"; - -#[cfg(target_os = "linux")] -const OODLE_LIB_NAME: &str = "liboo2corelinux64.so"; - pub(crate) fn command_definition() -> Command { Command::new("bundle") .subcommand_required(true) @@ -20,7 +14,7 @@ pub(crate) fn command_definition() -> Command { .arg( Arg::new("oodle") .long("oodle") - .default_value(OODLE_LIB_NAME) + .default_value(super::OODLE_LIB_NAME) .help( "The oodle library to load. This may either be:\n\ - A library name that will be searched for in the system's default paths.\n\ diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index d7a278e..96c7769 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -18,6 +18,12 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::EnvFilter; mod cmd { + #[cfg(target_os = "windows")] + const OODLE_LIB_NAME: &str = "oo2core_8_win64"; + + #[cfg(target_os = "linux")] + const OODLE_LIB_NAME: &str = "liboo2corelinux64.so"; + pub mod build; pub mod bundle; pub mod dictionary; @@ -27,6 +33,10 @@ mod cmd { pub mod watch; } +mod mods { + pub mod archive; +} + #[derive(Default, Deserialize, Serialize)] struct GlobalConfig { game_dir: Option, @@ -50,7 +60,7 @@ async fn main() -> Result<()> { .global(true) .value_parser(value_parser!(PathBuf)), ) - // .subcommand(cmd::build::command_definition()) + .subcommand(cmd::build::command_definition()) .subcommand(cmd::bundle::command_definition()) .subcommand(cmd::dictionary::command_definition()) .subcommand(cmd::murmur::command_definition()) diff --git a/crates/dtmt/src/mods/archive.rs b/crates/dtmt/src/mods/archive.rs new file mode 100644 index 0000000..b18cffa --- /dev/null +++ b/crates/dtmt/src/mods/archive.rs @@ -0,0 +1,97 @@ +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use color_eyre::eyre::{self, Context}; +use color_eyre::Result; +use sdk::murmur::Murmur64; +use sdk::Bundle; +use zip::ZipWriter; + +pub struct Archive { + name: String, + bundles: Vec, + mod_file: Option>, +} + +impl Archive { + pub fn new(name: String) -> Self { + Self { + name, + bundles: Vec::new(), + mod_file: None, + } + } + + pub fn add_bundle(&mut self, bundle: Bundle) { + self.bundles.push(bundle) + } + + pub fn add_mod_file(&mut self, content: Vec) { + self.mod_file = Some(content); + } + + pub fn write

(&self, ctx: &sdk::Context, path: P) -> Result<()> + where + P: AsRef, + { + let mod_file = self + .mod_file + .as_ref() + .ok_or_else(|| eyre::eyre!("Mod file is missing from mod archive"))?; + + let f = File::create(path.as_ref()).wrap_err_with(|| { + format!( + "failed to open file for reading: {}", + path.as_ref().display() + ) + })?; + let mut zip = ZipWriter::new(f); + + zip.add_directory(&self.name, Default::default())?; + + { + let mut name = path.as_ref().join(&self.name); + name.set_extension("mod"); + zip.start_file(name.to_string_lossy(), Default::default())?; + zip.write_all(mod_file)?; + } + + let path = PathBuf::from(&self.name); + let mut file_map = HashMap::new(); + + for bundle in self.bundles.iter() { + let bundle_name = bundle.name().clone(); + let bundle_path = PathBuf::from(&bundle_name); + + let map_entry: &mut HashSet<_> = file_map.entry(bundle_name).or_default(); + + for file in bundle.files() { + let bundle_path = bundle_path.join(file.base_name()); + map_entry.insert(bundle_path.to_string_lossy().to_string()); + } + + let name = Murmur64::hash(bundle.name().as_bytes()); + let path = path.join(name.to_string()); + + zip.start_file(path.to_string_lossy(), Default::default())?; + + let data = bundle.to_binary(ctx)?; + zip.write_all(&data)?; + } + + { + let data = serde_sjson::to_string(&file_map)?; + zip.start_file( + path.join("files.sjson").to_string_lossy(), + Default::default(), + )?; + zip.write_all(data.as_bytes())?; + } + + zip.finish()?; + + Ok(()) + } +} diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index 20fa0d0..c3ef0c0 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -1,7 +1,8 @@ use std::io::{Cursor, Read, Seek, Write}; +use std::path::Path; use color_eyre::eyre::Context; -use color_eyre::{Help, Result, SectionExt}; +use color_eyre::{eyre, Result}; use futures::future::join_all; use serde::Serialize; @@ -9,6 +10,8 @@ use crate::binary::sync::*; use crate::filetype::*; use crate::murmur::{HashGroup, Murmur64}; +use super::EntryHeader; + #[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] pub enum BundleFileType { Animation, @@ -161,6 +164,80 @@ impl BundleFileType { } } +impl std::str::FromStr for BundleFileType { + type Err = color_eyre::Report; + + fn from_str(s: &str) -> Result { + let val = match s { + "animation_curves" => BundleFileType::AnimationCurves, + "animation" => BundleFileType::Animation, + "apb" => BundleFileType::Apb, + "baked_lighting" => BundleFileType::BakedLighting, + "bik" => BundleFileType::Bik, + "blend_set" => BundleFileType::BlendSet, + "bones" => BundleFileType::Bones, + "chroma" => BundleFileType::Chroma, + "common_package" => BundleFileType::CommonPackage, + "config" => BundleFileType::Config, + "crypto" => BundleFileType::Crypto, + "data" => BundleFileType::Data, + "entity" => BundleFileType::Entity, + "flow" => BundleFileType::Flow, + "font" => BundleFileType::Font, + "ies" => BundleFileType::Ies, + "ini" => BundleFileType::Ini, + "input" => BundleFileType::Input, + "ivf" => BundleFileType::Ivf, + "keys" => BundleFileType::Keys, + "level" => BundleFileType::Level, + "lua" => BundleFileType::Lua, + "material" => BundleFileType::Material, + "mod" => BundleFileType::Mod, + "mouse_cursor" => BundleFileType::MouseCursor, + "nav_data" => BundleFileType::NavData, + "network_config" => BundleFileType::NetworkConfig, + "oodle_net" => BundleFileType::OddleNet, + "package" => BundleFileType::Package, + "particles" => BundleFileType::Particles, + "physics_properties" => BundleFileType::PhysicsProperties, + "render_config" => BundleFileType::RenderConfig, + "rt_pipeline" => BundleFileType::RtPipeline, + "scene" => BundleFileType::Scene, + "shader_library_group" => BundleFileType::ShaderLibraryGroup, + "shader_library" => BundleFileType::ShaderLibrary, + "shader" => BundleFileType::Shader, + "shading_environment_mapping" => BundleFileType::ShadingEnvionmentMapping, + "shading_environment" => BundleFileType::ShadingEnvironment, + "slug_album" => BundleFileType::SlugAlbum, + "slug" => BundleFileType::Slug, + "sound_environment" => BundleFileType::SoundEnvironment, + "spu_job" => BundleFileType::SpuJob, + "state_machine" => BundleFileType::StateMachine, + "static_pvs" => BundleFileType::StaticPVS, + "strings" => BundleFileType::Strings, + "surface_properties" => BundleFileType::SurfaceProperties, + "texture" => BundleFileType::Texture, + "timpani_bank" => BundleFileType::TimpaniBank, + "timpani_master" => BundleFileType::TimpaniMaster, + "tome" => BundleFileType::Tome, + "ugg" => BundleFileType::Ugg, + "unit" => BundleFileType::Unit, + "upb" => BundleFileType::Upb, + "vector_field" => BundleFileType::VectorField, + "wav" => BundleFileType::Wav, + "wwise_bank" => BundleFileType::WwiseBank, + "wwise_dep" => BundleFileType::WwiseDep, + "wwise_event" => BundleFileType::WwiseEvent, + "wwise_metadata" => BundleFileType::WwiseMetadata, + "wwise_stream" => BundleFileType::WwiseStream, + "xml" => BundleFileType::Xml, + s => eyre::bail!("Unknown type string '{}'", s), + }; + + Ok(val) + } +} + impl Serialize for BundleFileType { fn serialize(&self, serializer: S) -> Result where @@ -319,6 +396,13 @@ impl From for Murmur64 { } } +impl std::fmt::Display for BundleFileType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.ext_name()) + } +} + +#[derive(Debug)] struct BundleFileHeader { variant: u32, size: usize, @@ -392,8 +476,12 @@ pub struct BundleFile { } impl BundleFile { - #[tracing::instrument(name = "File::read", skip_all)] - pub fn from_reader(ctx: &crate::Context, r: &mut R) -> Result + #[tracing::instrument( + name = "File::read", + skip_all, + fields(name = %meta.name_hash, ext = %meta.extension_hash, flags = meta.flags) + )] + pub fn from_reader(ctx: &crate::Context, r: &mut R, meta: &EntryHeader) -> Result where R: Read + Seek, { @@ -401,8 +489,6 @@ impl BundleFile { let hash = Murmur64::from(r.read_u64()?); let name = ctx.lookup_hash(hash, HashGroup::Filename); - tracing::trace!(name, ?file_type); - let header_count = r.read_u32()? as usize; let mut headers = Vec::with_capacity(header_count); r.skip_u32(0)?; @@ -463,6 +549,15 @@ impl BundleFile { Ok(w.into_inner()) } + #[tracing::instrument(name = "File::from_sjson", skip(_sjson))] + pub async fn from_sjson(_file_type: BundleFileType, _sjson: S, _root: P) -> Result + where + P: AsRef + std::fmt::Debug, + S: AsRef, + { + todo!(); + } + pub fn base_name(&self) -> &String { &self.name } @@ -555,19 +650,17 @@ impl BundleFile { let res = match file_type { BundleFileType::Lua => lua::decompile(ctx, data).await, - BundleFileType::Package => package::decompile(ctx, data), + BundleFileType::Package => package::decompile(ctx, name.clone(), data), _ => { tracing::debug!("Can't decompile, unknown file type"); Ok(vec![UserFile::with_name(data.to_vec(), name.clone())]) } }; + let res = res.wrap_err_with(|| format!("failed to decompile file {name}")); match res { Ok(files) => files, Err(err) => { - let err = err - .wrap_err("failed to decompile file") - .with_section(|| name.header("File:")); tracing::error!("{:?}", err); vec![] } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 1926989..8791290 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -40,9 +40,9 @@ impl From for u32 { } } -struct EntryHeader { - name_hash: u64, - extension_hash: u64, +pub struct EntryHeader { + name_hash: Murmur64, + extension_hash: Murmur64, flags: u32, } @@ -52,8 +52,8 @@ impl EntryHeader { where R: Read + Seek, { - let extension_hash = r.read_u64()?; - let name_hash = r.read_u64()?; + let extension_hash = Murmur64::from(r.read_u64()?); + let name_hash = Murmur64::from(r.read_u64()?); let flags = r.read_u32()?; // NOTE: Known values so far: @@ -79,8 +79,8 @@ impl EntryHeader { where W: Write + Seek, { - w.write_u64(self.extension_hash)?; - w.write_u64(self.name_hash)?; + w.write_u64(*self.extension_hash)?; + w.write_u64(*self.name_hash)?; w.write_u32(self.flags)?; Ok(()) } @@ -89,12 +89,22 @@ impl EntryHeader { pub struct Bundle { format: BundleFormat, properties: [Murmur64; 32], - _headers: Vec, + headers: Vec, files: Vec, name: String, } impl Bundle { + pub fn new(name: String) -> Self { + Self { + name, + format: BundleFormat::F8, + properties: [0.into(); 32], + headers: Vec::new(), + files: Vec::new(), + } + } + pub fn get_name_from_path

(ctx: &crate::Context, path: P) -> String where P: AsRef, @@ -107,6 +117,17 @@ impl Bundle { .unwrap_or_else(|| path.display().to_string()) } + pub fn add_file(&mut self, file: BundleFile) { + let header = EntryHeader { + extension_hash: file.file_type().into(), + name_hash: Murmur64::hash(file.base_name().as_bytes()), + flags: 0x0, + }; + + self.files.push(file); + self.headers.push(header); + } + #[tracing::instrument(skip(ctx, binary), fields(len_binary = binary.as_ref().len()))] pub fn from_binary(ctx: &crate::Context, name: String, binary: B) -> Result where @@ -116,6 +137,7 @@ impl Bundle { let mut r = BufReader::new(Cursor::new(binary)); let format = r.read_u32().and_then(BundleFormat::try_from)?; + tracing::debug!(?format); if !matches!(format, BundleFormat::F7 | BundleFormat::F8) { return Err(eyre::eyre!("Unknown bundle format: {:?}", format)); @@ -130,9 +152,9 @@ impl Bundle { *prop = Murmur64::from(r.read_u64()?); } - let mut meta = Vec::with_capacity(num_entries); + let mut headers = Vec::with_capacity(num_entries); for _ in 0..num_entries { - meta.push(EntryHeader::from_reader(&mut r)?); + headers.push(EntryHeader::from_reader(&mut r)?); } let num_chunks = r.read_u32()? as usize; @@ -206,7 +228,8 @@ impl Bundle { let mut r = Cursor::new(decompressed); let mut files = Vec::with_capacity(num_entries); for i in 0..num_entries { - let file = BundleFile::from_reader(ctx, &mut r) + let meta = headers.get(i).unwrap(); + let file = BundleFile::from_reader(ctx, &mut r, meta) .wrap_err_with(|| format!("failed to read file {i}"))?; files.push(file); } @@ -214,7 +237,7 @@ impl Bundle { Ok(Self { name: bundle_name, format, - _headers: meta, + headers, files, properties, }) @@ -232,7 +255,7 @@ impl Bundle { w.write_u64(**prop)?; } - for meta in self._headers.iter() { + for meta in self.headers.iter() { meta.to_writer(&mut w)?; } @@ -269,7 +292,7 @@ impl Bundle { let chunks = unpacked_data.chunks(CHUNK_SIZE); - let oodle_lib = ctx.oodle.as_ref().unwrap(); + let oodle_lib = ctx.oodle.as_ref().expect("oodle library not defined"); let mut chunk_sizes = Vec::with_capacity(num_chunks); for chunk in chunks { diff --git a/lib/sdk/src/filetype/package.rs b/lib/sdk/src/filetype/package.rs index 4124068..a693c4b 100644 --- a/lib/sdk/src/filetype/package.rs +++ b/lib/sdk/src/filetype/package.rs @@ -1,63 +1,337 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::ops::{Deref, DerefMut}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; -use color_eyre::eyre::Context; +use color_eyre::eyre::{self, Context}; use color_eyre::Result; -use serde::Serialize; -use crate::binary::sync::ReadExt; +use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::{BundleFileType, UserFile}; use crate::murmur::{HashGroup, Murmur64}; -#[derive(Serialize)] -struct Package(HashMap>); +#[tracing::instrument(skip(_ctx))] +async fn resolve_wildcard

( + _ctx: &crate::Context, + wildcard: P, + t: Option, +) -> Result> +where + P: AsRef + std::fmt::Debug, +{ + let wildcard = wildcard.as_ref(); + + if wildcard.is_absolute() { + eyre::bail!( + "Path or wildcard must be relative. Got '{}'", + wildcard.display() + ); + } + + if !wildcard.ends_with("*") { + let mut path = wildcard.to_path_buf(); + + if let Some(t) = t { + path.push(t.ext_name()); + } + + return Ok(vec![path]); + } + + // let parent = wildcard.parent().unwrap_or(&ctx.project_dir); + + // let paths = Vec::new(); + // let dir = fs::read_dir(parent).await?; + + // while let Some(file) = dir.next_entry().await? { + // if let Some(ext) = file.file_name() + // } + + todo!(); +} + +type PackageType = HashMap>; +type PackageDefinition = HashMap>; + +#[derive(Default)] +pub struct Package { + name: String, + root: PathBuf, + inner: PackageType, +} impl Deref for Package { - type Target = HashMap>; + type Target = PackageType; fn deref(&self) -> &Self::Target { - &self.0 + &self.inner } } impl DerefMut for Package { fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 + &mut self.inner } } +#[tracing::instrument] +async fn glob_stream(pattern: PB, root: P) -> Result<(PathBuf, tokio::fs::ReadDir)> +where + PB: Into + std::fmt::Debug, + P: AsRef + std::fmt::Debug, +{ + let pattern: PathBuf = pattern.into(); + if pattern.is_absolute() { + eyre::bail!( + "Path in package definition must not be absolute. Got '{}'", + pattern.display() + ) + } + + let _is_pattern = pattern.ends_with("*"); + let _dir = pattern.parent().unwrap_or(root.as_ref()); + todo!(); + // let stream = fs::read_dir(dir).await?; + // Ok((dir.to_path_buf(), stream)) +} + impl Package { - fn new() -> Self { - Self(HashMap::new()) + fn len(&self) -> usize { + self.values().fold(0, |total, files| total + files.len()) + } + + fn add_file

(&mut self, t: BundleFileType, path: P) + where + P: Into, + { + self.entry(t).or_default().insert(path.into()); + } + + #[tracing::instrument("Package::from_sjson", skip(sjson), fields(sjson_len = sjson.as_ref().len()))] + pub async fn from_sjson(sjson: S, name: String, root: P) -> Result + where + P: AsRef + std::fmt::Debug, + S: AsRef, + { + let root = root.as_ref(); + let definition: PackageDefinition = serde_sjson::from_str(sjson.as_ref())?; + let mut inner: PackageType = Default::default(); + + for (ty, patterns) in definition.iter() { + if ty == "*" { + for pattern in patterns.iter() { + let (dir, mut stream) = glob_stream(pattern, root).await?; + + while let Some(entry) = stream.next_entry().await? { + let name = PathBuf::from(entry.file_name()); + let ext = if let Some(ext) = name.extension().and_then(|ext| ext.to_str()) { + match BundleFileType::from_str(ext) { + Ok(t) => t, + Err(_) => { + tracing::debug!( + "Skipping file with invalid extension: {}", + dir.join(name).display() + ); + continue; + } + } + } else { + tracing::debug!( + "Skipping file without extension: {}", + dir.join(name).display() + ); + continue; + }; + + inner.entry(ext).or_default().insert(dir.join(name)); + } + } + } else if let Ok(t) = BundleFileType::from_str(ty) { + for pattern in patterns.iter() { + let (dir, mut stream) = glob_stream(pattern, root).await?; + + while let Some(entry) = stream.next_entry().await? { + let name = PathBuf::from(entry.file_name()); + let ext = if let Some(ext) = name.extension().and_then(|ext| ext.to_str()) { + match BundleFileType::from_str(ext) { + Ok(t) => t, + Err(_) => { + tracing::debug!( + "Skipping file with invalid extension: {}", + dir.join(name).display() + ); + continue; + } + } + } else { + tracing::debug!( + "Skipping file without extension: {}", + dir.join(name).display() + ); + continue; + }; + + if t == ext { + inner.entry(ext).or_default().insert(dir.join(name)); + } + } + } + } else { + eyre::bail!("Unknown file type '{}'", ty); + }; + } + + let pkg = Self { + inner, + name, + root: root.to_path_buf(), + }; + + Ok(pkg) + } + + #[tracing::instrument("Package::to_sjson", skip(self), fields(types = self.inner.len(), files = self.len()))] + pub fn to_sjson(&self) -> Result { + let mut map: PackageDefinition = Default::default(); + + for (t, paths) in self.iter() { + for path in paths.iter() { + map.entry(t.ext_name()) + .or_default() + .insert(path.display().to_string()); + } + } + + serde_sjson::to_string(&map).wrap_err("failed to serialize Package to SJSON") + } + + #[tracing::instrument("Package::from_binary", skip(binary, ctx), fields(binary_len = binary.as_ref().len()))] + pub fn from_binary(ctx: &crate::Context, name: String, binary: B) -> Result + where + B: AsRef<[u8]>, + { + let mut r = Cursor::new(binary.as_ref()); + + // TODO: Figure out what this is + let unknown = r.read_u32()?; + if unknown != 0x2b { + tracing::warn!("Unknown u32 header. Expected 0x2b, got: {unknown:#08X} ({unknown})"); + } + + let file_count = r.read_u32()? as usize; + let mut inner: PackageType = Default::default(); + + for _ in 0..file_count { + let t = BundleFileType::from(r.read_u64()?); + let hash = Murmur64::from(r.read_u64()?); + let path = ctx.lookup_hash(*hash, HashGroup::Filename); + inner.entry(t).or_default().insert(PathBuf::from(path)); + } + + let pkg = Self { + inner, + name, + root: PathBuf::new(), + }; + + Ok(pkg) + } + + #[tracing::instrument("Package::to_binary", skip(self), fields(types = self.inner.len(), files = self.len()))] + pub fn to_binary(&self) -> Result> { + let mut w = Cursor::new(Vec::new()); + + // TODO: Figure out what this is + w.write_u32(0x2b)?; + w.write_u32(self.values().flatten().count() as u32)?; + + for (t, paths) in self.iter() { + for path in paths.iter() { + w.write_u64(*t.hash())?; + + let hash = Murmur64::hash(path.to_string_lossy().as_bytes()); + w.write_u64(*hash)?; + } + } + + Ok(w.into_inner()) } } -#[tracing::instrument(skip_all)] -pub fn decompile(ctx: &crate::Context, binary: B) -> Result> +#[tracing::instrument(skip(ctx, data))] +pub fn decompile(ctx: &crate::Context, name: String, data: B) -> Result> where B: AsRef<[u8]>, { - let mut r = Cursor::new(binary.as_ref()); - // TODO: Figure out what this is - let unknown = r.read_u32()?; - if unknown != 0x2b { - tracing::warn!("Unknown u32 header. Expected 0x2b, got: {unknown:#08X} ({unknown})"); - } - - let file_count = r.read_u32()? as usize; - let mut package = Package::new(); - - for i in 0..file_count { - let t = BundleFileType::from(r.read_u64()?); - let hash = Murmur64::from(r.read_u64()?); - let name = ctx.lookup_hash(hash, HashGroup::Filename); - - tracing::trace!(index = i, r"type" = ?t, %hash, name, "Package entry"); - - package.entry(t).or_insert_with(Vec::new).push(name); - } - - let s = serde_sjson::to_string(&package.0).wrap_err("failed to serialize Package to SJSON")?; + let pkg = Package::from_binary(ctx, name, data)?; + let s = pkg.to_sjson()?; Ok(vec![UserFile::new(s.into_bytes())]) } + +// #[tracing::instrument(skip_all)] +// pub fn compile(_ctx: &crate::Context, data: String) -> Result> { +// let pkg = Package::from_sjson(data)?; +// pkg.to_binary() +// } + +#[cfg(test)] +mod test { + use crate::BundleFileType; + + use super::Package; + + #[test] + fn to_binary_empty_package() { + let pkg = Package::default(); + + assert_eq!( + pkg.to_binary().unwrap(), + vec![0x2b, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0] + ); + } + + #[test] + fn to_binary_single_file() { + let mut pkg = Package::default(); + pkg.add_file(BundleFileType::Lua, "lua"); + + assert_eq!( + pkg.to_binary().unwrap(), + vec![ + 0x2b, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xe2, 0x17, 0xd1, 0x2c, 0xfa, 0x8d, 0x4e, + 0xa1, 0xe2, 0x17, 0xd1, 0x2c, 0xfa, 0x8d, 0x4e, 0xa1, + ] + ); + } + + #[test] + #[should_panic(expected = "not yet implemented")] + fn to_sjson_empty_package() { + todo!(); + } + + #[test] + #[should_panic(expected = "not yet implemented")] + fn to_sjson_single_file() { + todo!(); + } + + #[test] + #[should_panic(expected = "not yet implemented")] + fn to_sjson_multiple_file_types() { + todo!(); + } + + #[tokio::test] + async fn from_sjson_empty() { + let root = std::env::current_dir().unwrap(); + let sjson = ""; + let name = String::new(); + + assert_eq!( + Package::from_sjson(sjson, name, root).await.unwrap().inner, + Default::default() + ); + } +} diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 13ce02b..2ecbfd7 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -3,11 +3,11 @@ mod binary; mod bundle; mod context; -mod filetype; +pub mod filetype; pub mod murmur; mod oodle; pub use bundle::decompress; -pub use bundle::{Bundle, BundleFile}; +pub use bundle::{Bundle, BundleFile, BundleFileType}; pub use context::Context; pub use oodle::Oodle;