From a53ce85f38cf22b5f4d18cef1c430d1520545160 Mon Sep 17 00:00:00 2001 From: projectmoon Date: Sat, 24 Oct 2020 13:46:06 +0000 Subject: [PATCH] Add db migration support, change variables schema. This is a bit of a large commit that adds basic database migration support. It also alters the way user variables are stored in a way requiring manual migration of existing data. The first automated migration adds variable count in a new place. --- Cargo.lock | 398 +++++++++++++++++++++++++-------- Cargo.toml | 1 + src/bin/dicebot.rs | 2 + src/config.rs | 13 ++ src/db.rs | 52 ++++- src/db/data_migrations.rs | 22 ++ src/db/errors.rs | 26 +++ src/db/migrations.rs | 54 +++++ src/db/schema.rs | 15 +- src/db/variables.rs | 94 +++++--- src/db/variables/migrations.rs | 68 ++++++ 11 files changed, 619 insertions(+), 126 deletions(-) create mode 100644 src/db/data_migrations.rs create mode 100644 src/db/migrations.rs create mode 100644 src/db/variables/migrations.rs diff --git a/Cargo.lock b/Cargo.lock index 27d32cd..8e5e42b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -70,9 +70,9 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f9db3b38af870bf7e5cc649167533b493928e50744e2c30ae350230b414670" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -81,9 +81,9 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -92,7 +92,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -106,6 +106,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -193,7 +199,8 @@ dependencies = [ "matrix-sdk", "nom", "olm-sys", - "rand", + "phf", + "rand 0.7.3", "serde", "sled", "tempfile", @@ -216,6 +223,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "cloudabi" version = "0.1.0" @@ -308,7 +324,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cfg-if 0.1.10", "crossbeam-utils", "lazy_static", @@ -334,7 +350,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cfg-if 0.1.10", "lazy_static", ] @@ -457,6 +473,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -536,9 +558,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -777,7 +799,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ - "autocfg", + "autocfg 1.0.1", "hashbrown", ] @@ -983,8 +1005,8 @@ name = "matrix-sdk-common-macros" version = "0.1.0" source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=master#6872cc717b8b7746ea4c4bec8d143e8653f965ed" dependencies = [ - "quote", - "syn", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1027,7 +1049,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" dependencies = [ - "autocfg", + "autocfg 1.0.1", ] [[package]] @@ -1213,7 +1235,7 @@ version = "0.9.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cc", "libc", "pkg-config", @@ -1238,7 +1260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ "cfg-if 0.1.10", - "cloudabi", + "cloudabi 0.1.0", "instant", "libc", "redox_syscall", @@ -1252,6 +1274,48 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_macros" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb45e833315153371697760dad1831da99ce41884162320305e4f123ca3fe37" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "0.4.27" @@ -1276,9 +1340,9 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1287,9 +1351,9 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1337,13 +1401,22 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + [[package]] name = "proc-macro2" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" dependencies = [ - "unicode-xid", + "unicode-xid 0.2.1", ] [[package]] @@ -1352,13 +1425,41 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + [[package]] name = "quote" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" dependencies = [ - "proc-macro2", + "proc-macro2 1.0.24", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", ] [[package]] @@ -1369,9 +1470,19 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ "getrandom 0.1.15", "libc", - "rand_chacha", - "rand_core", - "rand_hc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", ] [[package]] @@ -1381,9 +1492,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.5.1", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -1393,13 +1519,84 @@ dependencies = [ "getrandom 0.1.15", ] +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi 0.0.3", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", ] [[package]] @@ -1515,9 +1712,9 @@ version = "0.17.0-alpha.1" source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30" dependencies = [ "proc-macro-crate", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1574,9 +1771,9 @@ version = "0.22.0-alpha.1" source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30" dependencies = [ "proc-macro-crate", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1595,10 +1792,10 @@ name = "ruma-identifiers-macros" version = "0.17.4" source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.7", "ruma-identifiers-validation", - "syn", + "syn 1.0.48", ] [[package]] @@ -1724,9 +1921,9 @@ version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -1768,6 +1965,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + [[package]] name = "slab" version = "0.4.2" @@ -1865,10 +2068,10 @@ dependencies = [ "futures", "heck", "lazy_static", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.7", "sqlx-core", - "syn", + "syn 1.0.48", "tokio", "url", ] @@ -1908,11 +2111,11 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.7", "serde", "serde_derive", - "syn", + "syn 1.0.48", ] [[package]] @@ -1922,13 +2125,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" dependencies = [ "base-x", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.7", "serde", "serde_derive", "serde_json", "sha1", - "syn", + "syn 1.0.48", ] [[package]] @@ -1953,9 +2156,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5" dependencies = [ "heck", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", ] [[package]] @@ -1964,9 +2178,9 @@ version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.7", + "unicode-xid 0.2.1", ] [[package]] @@ -1975,10 +2189,10 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2", - "quote", - "syn", - "unicode-xid", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", + "unicode-xid 0.2.1", ] [[package]] @@ -1989,7 +2203,7 @@ checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" dependencies = [ "cfg-if 0.1.10", "libc", - "rand", + "rand 0.7.3", "redox_syscall", "remove_dir_all", "winapi 0.3.9", @@ -2019,9 +2233,9 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -2065,10 +2279,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa" dependencies = [ "proc-macro-hack", - "proc-macro2", - "quote", + "proc-macro2 1.0.24", + "quote 1.0.7", "standback", - "syn", + "syn 1.0.48", ] [[package]] @@ -2107,9 +2321,9 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -2170,9 +2384,9 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", ] [[package]] @@ -2233,6 +2447,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -2262,7 +2482,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" dependencies = [ - "rand", + "rand 0.7.3", ] [[package]] @@ -2314,9 +2534,9 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", "wasm-bindgen-shared", ] @@ -2338,7 +2558,7 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" dependencies = [ - "quote", + "quote 1.0.7", "wasm-bindgen-macro-support", ] @@ -2348,9 +2568,9 @@ version = "0.2.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2449,8 +2669,8 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb" dependencies = [ - "proc-macro2", - "syn", + "proc-macro2 1.0.24", + "syn 1.0.48", "synstructure", ] @@ -2469,8 +2689,8 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16" dependencies = [ - "proc-macro2", - "quote", - "syn", + "proc-macro2 1.0.24", + "quote 1.0.7", + "syn 1.0.48", "synstructure", ] diff --git a/Cargo.toml b/Cargo.toml index a1d53a2..125bef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ sled = "0.34" zerocopy = "0.3" byteorder = "1.3" futures = "0.3" +phf = { version = "0.7", features = ["macros"] } olm-sys = "1.0" matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk", rev = "master" } diff --git a/src/bin/dicebot.rs b/src/bin/dicebot.rs index 718c71b..39f7077 100644 --- a/src/bin/dicebot.rs +++ b/src/bin/dicebot.rs @@ -29,6 +29,8 @@ async fn run() -> Result<(), BotError> { let db = Database::new(&cfg.database_path())?; let state = Arc::new(RwLock::new(DiceBotState::new(&cfg))); + db.migrate(cfg.migration_version())?; + match DiceBot::new(&cfg, &state, &db) { Ok(bot) => bot.run().await?, Err(e) => println!("Error connecting: {:?}", e), diff --git a/src/config.rs b/src/config.rs index 1590813..c5e3a3a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -4,6 +4,10 @@ use std::fs; use std::path::PathBuf; use thiserror::Error; +/// Shortcut to defining db migration versions. Will probably +/// eventually be moved to a config file. +const MIGRATION_VERSION: u32 = 1; + #[derive(Error, Debug)] pub enum ConfigError { #[error("i/o error: {0}")] @@ -124,6 +128,15 @@ impl Config { .unwrap_or_else(|| db_path_from_env()) } + /// The current migration version we expect of the database. If + /// this number is higher than the one in the database, we will + /// execute migrations to update the data. + #[inline] + #[must_use] + pub fn migration_version(&self) -> u32 { + MIGRATION_VERSION + } + /// Figure out the allowed oldest message age, in seconds. This will /// be the defined oldest message age in the bot config, if the bot /// configuration and associated "oldest_message_age" setting are diff --git a/src/db.rs b/src/db.rs index f385cb9..9dd0650 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,9 +1,13 @@ -use crate::db::errors::DataError; +use crate::db::errors::{DataError, MigrationError}; +use crate::db::migrations::{get_migration_version, Migrations}; use crate::db::variables::Variables; +use log::info; use sled::Db; use std::path::Path; +pub mod data_migrations; pub mod errors; +pub mod migrations; pub mod schema; pub mod variables; @@ -11,19 +15,59 @@ pub mod variables; pub struct Database { db: Db, pub(crate) variables: Variables, - //rooms: Tree, + pub(crate) migrations: Migrations, } impl Database { pub fn new>(path: P) -> Result { let db = sled::open(path)?; let variables = db.open_tree("variables")?; - //let rooms = db.open_tree("rooms")?; + let migrations = db.open_tree("migrations")?; Ok(Database { db: db.clone(), variables: Variables(variables), - //rooms: rooms, + migrations: Migrations(migrations), }) } + + pub fn migrate(&self, to_version: u32) -> Result<(), DataError> { + //get version from db + let db_version = get_migration_version(&self)?; + + if db_version < to_version { + info!( + "Migrating database from version {} to version {}", + db_version, to_version + ); + //if db version < to_version, proceed + //produce range of db_version+1 .. to_version (inclusive) + let versions_to_run: Vec = ((db_version + 1)..=to_version).collect(); + let migrations = data_migrations::get_migrations(&versions_to_run)?; + + //execute each closure. + for (version, migration_func) in versions_to_run.iter().zip(migrations) { + //This needs to be transactional on migrations + //keyspace. abort on migration func error. + + info!("Applying migration: {}", version); + match migration_func(&self) { + Ok(_) => Ok(()), + Err(e) => Err(e), + }?; + + self.migrations.set_migration_version(*version)?; + } + + info!("Done applying migrations."); + Ok(()) + } else if db_version > to_version { + //if db version > to_version, cannot downgrade error + Err(MigrationError::CannotDowngrade.into()) + } else { + //if db version == to_version, do nothing + info!("No database migrations needed."); + Ok(()) + } + } } diff --git a/src/db/data_migrations.rs b/src/db/data_migrations.rs new file mode 100644 index 0000000..222dc7d --- /dev/null +++ b/src/db/data_migrations.rs @@ -0,0 +1,22 @@ +use crate::db::errors::{DataError, MigrationError}; +use crate::db::Database; +use phf::phf_map; + +pub(super) type DataMigration = fn(&Database) -> Result<(), DataError>; + +static MIGRATIONS: phf::Map = phf_map! { + 1u32 => super::variables::migrations::migration1, +}; + +pub fn get_migrations(versions: &[u32]) -> Result, MigrationError> { + let mut migrations: Vec = vec![]; + + for version in versions { + match MIGRATIONS.get(version) { + Some(func) => migrations.push(*func), + None => return Err(MigrationError::MigrationNotFound(*version)), + } + } + + Ok(migrations) +} diff --git a/src/db/errors.rs b/src/db/errors.rs index 9294e19..b1f9ab0 100644 --- a/src/db/errors.rs +++ b/src/db/errors.rs @@ -1,6 +1,18 @@ use sled::transaction::{TransactionError, UnabortableTransactionError}; use thiserror::Error; +#[derive(Error, Debug)] +pub enum MigrationError { + #[error("cannot downgrade to an older database version")] + CannotDowngrade, + + #[error("migration for version {0} not defined")] + MigrationNotFound(u32), + + #[error("migration failed: {0}")] + MigrationFailed(String), +} + //TODO better combining of key and value in certain errors (namely //I32SchemaViolation). #[derive(Error, Debug)] @@ -22,6 +34,9 @@ pub enum DataError { #[error("unabortable transaction error: {0}")] UnabortableTransactionError(#[from] UnabortableTransactionError), + + #[error("data migration error: {0}")] + MigrationError(#[from] MigrationError), } /// This From implementation is necessary to deal with the recursive @@ -40,3 +55,14 @@ impl From> for DataError { } } } + +// impl From> for DataError { +// fn from(error: ConflictableTransactionError) -> Self { +// match error { +// ConflictableTransactionError::Abort(data_err) => data_err, +// ConflictableTransactionError::Storage(storage_err) => { +// DataError::TransactionError(TransactionError::Storage(storage_err)) +// } +// } +// } +// } diff --git a/src/db/migrations.rs b/src/db/migrations.rs new file mode 100644 index 0000000..59f4fdf --- /dev/null +++ b/src/db/migrations.rs @@ -0,0 +1,54 @@ +use crate::db::errors::DataError; +use crate::db::schema::convert_u32; +use crate::db::Database; +use byteorder::LittleEndian; +use sled::Tree; +use zerocopy::byteorder::U32; +use zerocopy::AsBytes; + +//This file is for controlling the migration info stored in the +//database, not actually running migrations. + +#[derive(Clone)] +pub struct Migrations(pub(super) Tree); + +const COLON: &'static [u8] = b":"; +const METADATA_SPACE: &'static str = "metadata"; +const MIGRATION_KEY: &'static str = "migration_version"; + +fn to_key(keyspace: &str, key_name: &str) -> Vec { + let mut key = vec![]; + key.extend_from_slice(keyspace.as_bytes()); + key.extend_from_slice(COLON); + key.extend_from_slice(key_name.as_bytes()); + key +} + +fn metadata_key(key_name: &str) -> Vec { + to_key(METADATA_SPACE, key_name) +} + +impl Migrations { + pub(super) fn set_migration_version(&self, version: u32) -> Result<(), DataError> { + //Rust cannot type infer this transaction + let result: Result<_, sled::transaction::TransactionError> = + self.0.transaction(|tx| { + let key = metadata_key(MIGRATION_KEY); + let db_value: U32 = U32::new(version); + tx.insert(key, db_value.as_bytes())?; + Ok(()) + }); + + result?; + + Ok(()) + } +} + +pub(super) fn get_migration_version(db: &Database) -> Result { + let key = metadata_key(MIGRATION_KEY); + match db.migrations.0.get(key)? { + Some(bytes) => convert_u32(&bytes), + None => Ok(0), + } +} diff --git a/src/db/schema.rs b/src/db/schema.rs index 893c581..61eec0a 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -1,6 +1,6 @@ use crate::db::errors::DataError; use byteorder::LittleEndian; -use zerocopy::byteorder::I32; +use zerocopy::byteorder::{I32, U32}; use zerocopy::LayoutVerified; /// User variables are stored as little-endian 32-bit integers in the @@ -8,6 +8,8 @@ use zerocopy::LayoutVerified; /// read. type LittleEndianI32Layout<'a> = LayoutVerified<&'a [u8], I32>; +type LittleEndianU32Layout<'a> = LayoutVerified<&'a [u8], U32>; + /// Convert bytes to an i32 with zero-copy deserialization. An error /// is returned if the bytes do not represent an i32. pub(super) fn convert_i32(raw_value: &[u8]) -> Result { @@ -20,3 +22,14 @@ pub(super) fn convert_i32(raw_value: &[u8]) -> Result { Err(DataError::I32SchemaViolation) } } + +pub(super) fn convert_u32(raw_value: &[u8]) -> Result { + let layout = LittleEndianU32Layout::new_unaligned(raw_value.as_ref()); + + if let Some(layout) = layout { + let value: U32 = *layout; + Ok(value.get()) + } else { + Err(DataError::I32SchemaViolation) + } +} diff --git a/src/db/variables.rs b/src/db/variables.rs index bf5b4fe..8871b7d 100644 --- a/src/db/variables.rs +++ b/src/db/variables.rs @@ -1,46 +1,71 @@ use crate::db::errors::DataError; use crate::db::schema::convert_i32; use byteorder::LittleEndian; -use sled::transaction::abort; -use sled::transaction::TransactionalTree; +use sled::transaction::{abort, TransactionalTree}; use sled::Tree; use std::collections::HashMap; +use std::convert::From; use std::str; use zerocopy::byteorder::I32; use zerocopy::AsBytes; -const METADATA_KEY: &'static str = "metadata"; +pub(super) mod migrations; + +const METADATA_SPACE: &'static [u8] = b"metadata"; +const VARIABLE_SPACE: &'static [u8] = b"variables"; + const VARIABLE_COUNT_KEY: &'static str = "variable_count"; #[derive(Clone)] -pub struct Variables(pub(crate) Tree); +pub struct Variables(pub(super) Tree); -fn to_key(room_id: &str, username: &str, variable_name: &str) -> Vec { - let mut key = vec![]; - key.extend_from_slice(room_id.as_bytes()); - key.extend_from_slice(username.as_bytes()); - key.extend_from_slice(variable_name.as_bytes()); - key +//TODO at least some of these will probalby move elsewhere. + +fn space_prefix>>(space: &[u8], delineator: D) -> Vec { + let mut metadata_prefix = vec![]; + metadata_prefix.extend_from_slice(space); + metadata_prefix.push(0xff); + let delineator = delineator.into(); + + if delineator.len() > 0 { + metadata_prefix.extend_from_slice(delineator.as_bytes()); + metadata_prefix.push(0xff); + } + + metadata_prefix } -fn metadata_key(room_id: &str, username: &str, metadata_key: &str) -> Vec { - let mut key = vec![]; - key.extend_from_slice(room_id.as_bytes()); - key.extend_from_slice(METADATA_KEY.as_bytes()); - key.extend_from_slice(username.as_bytes()); - key.extend_from_slice(metadata_key.as_bytes()); - key +fn metadata_space_prefix>>(delineator: D) -> Vec { + space_prefix(METADATA_SPACE, delineator) } -fn room_variable_count_key(room_id: &str, username: &str) -> Vec { - metadata_key(room_id, username, VARIABLE_COUNT_KEY) +fn metadata_space_key>>(delineator: D, key_name: &str) -> Vec { + let mut metadata_key = metadata_space_prefix(delineator); + metadata_key.extend_from_slice(key_name.as_bytes()); + metadata_key } -fn to_prefix(room_id: &str, username: &str) -> Vec { - let mut prefix = vec![]; - prefix.extend_from_slice(room_id.as_bytes()); - prefix.extend_from_slice(username.as_bytes()); - prefix +fn variables_space_prefix>>(delineator: D) -> Vec { + space_prefix(VARIABLE_SPACE, delineator) +} + +fn variables_space_key>>(delineator: D, key_name: &str) -> Vec { + let mut metadata_key = variables_space_prefix(delineator); + metadata_key.extend_from_slice(key_name.as_bytes()); + metadata_key +} + +/// Delineator for keeping track of a key by room ID and username. +struct RoomAndUser<'a>(&'a str, &'a str); + +impl<'a> From> for Vec { + fn from(value: RoomAndUser<'a>) -> Vec { + let mut bytes = vec![]; + bytes.extend_from_slice(value.0.as_bytes()); + bytes.push(0xff); + bytes.extend_from_slice(value.1.as_bytes()); + bytes + } } /// Use a transaction to atomically alter the count of variables in @@ -51,7 +76,8 @@ fn alter_room_variable_count( username: &str, amount: i32, ) -> Result { - let key = room_variable_count_key(room_id, username); + let key = metadata_space_key(RoomAndUser(room_id, username), VARIABLE_COUNT_KEY); + let mut new_count = match variables.get(&key)? { Some(bytes) => convert_i32(&bytes)? + amount, None => amount, @@ -72,7 +98,7 @@ impl Variables { room_id: &str, username: &str, ) -> Result, DataError> { - let prefix = to_prefix(&room_id, &username); + let prefix = variables_space_prefix(RoomAndUser(room_id, username)); let prefix_len: usize = prefix.len(); let variables: Result, DataError> = self @@ -94,7 +120,9 @@ impl Variables { } pub fn get_variable_count(&self, room_id: &str, username: &str) -> Result { - let key = room_variable_count_key(room_id, username); + let delineator = RoomAndUser(room_id, username); + let key = metadata_space_key(delineator, VARIABLE_COUNT_KEY); + if let Some(raw_value) = self.0.get(&key)? { convert_i32(&raw_value) } else { @@ -108,12 +136,12 @@ impl Variables { username: &str, variable_name: &str, ) -> Result { - let key = to_key(room_id, username, variable_name); + let key = variables_space_key(RoomAndUser(room_id, username), variable_name); if let Some(raw_value) = self.0.get(&key)? { convert_i32(&raw_value) } else { - Err(DataError::KeyDoesNotExist(String::from_utf8(key).unwrap())) + Err(DataError::KeyDoesNotExist(variable_name.to_owned())) } } @@ -126,7 +154,7 @@ impl Variables { ) -> Result<(), DataError> { self.0 .transaction(|tx| { - let key = to_key(room_id, username, variable_name); + let key = variables_space_key(RoomAndUser(room_id, username), variable_name); let db_value: I32 = I32::new(value); let old_value = tx.insert(key, db_value.as_bytes())?; @@ -151,14 +179,16 @@ impl Variables { ) -> Result<(), DataError> { self.0 .transaction(|tx| { - let key = to_key(room_id, username, variable_name); + let key = variables_space_key(RoomAndUser(room_id, username), variable_name); + + //TODO why does tx.remove require moving the key? if let Some(_) = tx.remove(key.clone())? { match alter_room_variable_count(&tx, room_id, username, -1) { Err(e) => abort(e), _ => Ok(()), } } else { - abort(DataError::KeyDoesNotExist(String::from_utf8(key).unwrap())) + abort(DataError::KeyDoesNotExist(variable_name.to_owned())) } }) .map_err(|e| e.into()) diff --git a/src/db/variables/migrations.rs b/src/db/variables/migrations.rs new file mode 100644 index 0000000..1333d75 --- /dev/null +++ b/src/db/variables/migrations.rs @@ -0,0 +1,68 @@ +use super::*; +use crate::db::errors::{DataError, MigrationError}; +use crate::db::Database; +use byteorder::LittleEndian; +use sled::transaction::TransactionError; +use sled::Batch; +use zerocopy::byteorder::U32; +use zerocopy::AsBytes; + +//TODO we will make this set variable count properly. +pub(in crate::db) fn migration1(db: &Database) -> Result<(), DataError> { + let tree = &db.variables.0; + let prefix = variables_space_prefix(""); + + //Extract a vec of tuples, consisting of room id + username. + let results: Vec<(String, String)> = tree + .scan_prefix(prefix) + .map(|entry| { + if let Ok((key, _)) = entry { + let keys: Vec> = key + .split(|&b| b == 0xff) + .map(|b| str::from_utf8(b)) + .collect(); + + if let &[_, Ok(room_id), Ok(username), Ok(_variable)] = keys.as_slice() { + Ok((room_id.to_owned(), username.to_owned())) + } else { + Err(MigrationError::MigrationFailed( + "a key violates utf8 schema".to_string(), + )) + } + } else { + Err(MigrationError::MigrationFailed( + "encountered unexpected key".to_string(), + )) + } + }) + .collect::, MigrationError>>()?; + + let counts: HashMap<(String, String), u32> = + results + .into_iter() + .fold(HashMap::new(), |mut count_map, room_and_user| { + let count = count_map.entry(room_and_user).or_insert(0); + *count += 1; + count_map + }); + + //Start a transaction on the variables tree. + //Delete the old variable_count variable if exists. + //Add variable count according to new schema. + let tx_result: Result<_, TransactionError> = db.variables.0.transaction(|tx_vars| { + let batch = counts.iter().fold(Batch::default(), |mut batch, entry| { + let key = + variables_space_key(RoomAndUser(&(entry.0).0, &(entry.0).1), VARIABLE_COUNT_KEY); + + let db_value: U32 = U32::new(*entry.1); + batch.insert(key, db_value.as_bytes()); + batch + }); + + tx_vars.apply_batch(&batch)?; + Ok(()) + }); + + tx_result?; //For some reason, it cannot infer the type + Ok(()) +}