Add db migration support, change variables schema.

This is a bit of a large commit that adds basic database migration
support. It also alters the way user variables are stored in a way
requiring manual migration of existing data. The first automated
migration adds variable count in a new place.

Cargo.lock

@@ -70,9 +70,9 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "25f9db3b38af870bf7e5cc649167533b493928e50744e2c30ae350230b414670"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -81,9 +81,9 @@ version = "0.1.41"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -92,7 +92,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3410529e8288c463bedb5930f82833bc0c90e5d2fe639a56582a4d09220b281"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
 ]
 
 [[package]]
@@ -106,6 +106,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "autocfg"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
+
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -193,7 +199,8 @@ dependencies = [
  "matrix-sdk",
  "nom",
  "olm-sys",
- "rand",
+ "phf",
+ "rand 0.7.3",
  "serde",
  "sled",
  "tempfile",
@@ -216,6 +223,15 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "cloudabi"
 version = "0.1.0"
@@ -308,7 +324,7 @@ version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "cfg-if 0.1.10",
  "crossbeam-utils",
  "lazy_static",
@@ -334,7 +350,7 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "cfg-if 0.1.10",
  "lazy_static",
 ]
@@ -457,6 +473,12 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
 [[package]]
 name = "fuchsia-zircon"
 version = "0.3.3"
@@ -536,9 +558,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe"
 dependencies = [
  "proc-macro-hack",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -777,7 +799,7 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "hashbrown",
 ]
 
@@ -983,8 +1005,8 @@ name = "matrix-sdk-common-macros"
 version = "0.1.0"
 source = "git+https://github.com/matrix-org/matrix-rust-sdk?rev=master#6872cc717b8b7746ea4c4bec8d143e8653f965ed"
 dependencies = [
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1027,7 +1049,7 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
 ]
 
 [[package]]
@@ -1213,7 +1235,7 @@ version = "0.9.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
 dependencies = [
- "autocfg",
+ "autocfg 1.0.1",
  "cc",
  "libc",
  "pkg-config",
@@ -1238,7 +1260,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
 dependencies = [
  "cfg-if 0.1.10",
- "cloudabi",
+ "cloudabi 0.1.0",
  "instant",
  "libc",
  "redox_syscall",
@@ -1252,6 +1274,48 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "phf"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18"
+dependencies = [
+ "phf_macros",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662"
+dependencies = [
+ "phf_shared",
+ "rand 0.6.5",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdb45e833315153371697760dad1831da99ce41884162320305e4f123ca3fe37"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2 0.4.30",
+ "quote 0.6.13",
+ "syn 0.15.44",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.7.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project"
 version = "0.4.27"
@@ -1276,9 +1340,9 @@ version = "0.4.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1287,9 +1351,9 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1337,13 +1401,22 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a"
 
+[[package]]
+name = "proc-macro2"
+version = "0.4.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
+dependencies = [
+ "unicode-xid 0.1.0",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
 dependencies = [
- "unicode-xid",
+ "unicode-xid 0.2.1",
 ]
 
 [[package]]
@@ -1352,13 +1425,41 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quote"
+version = "0.6.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
+dependencies = [
+ "proc-macro2 0.4.30",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
+]
+
+[[package]]
+name = "rand"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
+dependencies = [
+ "autocfg 0.1.7",
+ "libc",
+ "rand_chacha 0.1.1",
+ "rand_core 0.4.2",
+ "rand_hc 0.1.0",
+ "rand_isaac",
+ "rand_jitter",
+ "rand_os",
+ "rand_pcg",
+ "rand_xorshift",
+ "winapi 0.3.9",
 ]
 
 [[package]]
@@ -1369,9 +1470,19 @@ checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
 dependencies = [
  "getrandom 0.1.15",
  "libc",
- "rand_chacha",
+ "rand_chacha 0.2.2",
- "rand_core",
+ "rand_core 0.5.1",
- "rand_hc",
+ "rand_hc 0.2.0",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
+dependencies = [
+ "autocfg 0.1.7",
+ "rand_core 0.3.1",
 ]
 
 [[package]]
@@ -1381,9 +1492,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
 [[package]]
 name = "rand_core"
 version = "0.5.1"
@@ -1393,13 +1519,84 @@ dependencies = [
  "getrandom 0.1.15",
 ]
 
+[[package]]
+name = "rand_hc"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "rand_hc"
 version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
 dependencies = [
- "rand_core",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_isaac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rand_jitter"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
+dependencies = [
+ "libc",
+ "rand_core 0.4.2",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "rand_os"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
+dependencies = [
+ "cloudabi 0.0.3",
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.4.2",
+ "rdrand",
+ "winapi 0.3.9",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
+dependencies = [
+ "autocfg 0.1.7",
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
 ]
 
 [[package]]
@@ -1515,9 +1712,9 @@ version = "0.17.0-alpha.1"
 source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30"
 dependencies = [
  "proc-macro-crate",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1574,9 +1771,9 @@ version = "0.22.0-alpha.1"
 source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30"
 dependencies = [
  "proc-macro-crate",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1595,10 +1792,10 @@ name = "ruma-identifiers-macros"
 version = "0.17.4"
 source = "git+https://github.com/ruma/ruma?rev=409fbcc9d745fb7290327cb7f5defc714229ab30#409fbcc9d745fb7290327cb7f5defc714229ab30"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
  "ruma-identifiers-validation",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1724,9 +1921,9 @@ version = "1.0.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1768,6 +1965,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "siphasher"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
+
 [[package]]
 name = "slab"
 version = "0.4.2"
@@ -1865,10 +2068,10 @@ dependencies = [
  "futures",
  "heck",
  "lazy_static",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
  "sqlx-core",
- "syn",
+ "syn 1.0.48",
  "tokio",
  "url",
 ]
@@ -1908,11 +2111,11 @@ version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
  "serde",
  "serde_derive",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1922,13 +2125,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11"
 dependencies = [
  "base-x",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
  "serde",
  "serde_derive",
  "serde_json",
  "sha1",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -1953,9 +2156,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5"
 dependencies = [
  "heck",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
+]
+
+[[package]]
+name = "syn"
+version = "0.15.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
+dependencies = [
+ "proc-macro2 0.4.30",
+ "quote 0.6.13",
+ "unicode-xid 0.1.0",
 ]
 
 [[package]]
@@ -1964,9 +2178,9 @@ version = "1.0.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "unicode-xid",
+ "unicode-xid 0.2.1",
 ]
 
 [[package]]
@@ -1975,10 +2189,10 @@ version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
- "unicode-xid",
+ "unicode-xid 0.2.1",
 ]
 
 [[package]]
@@ -1989,7 +2203,7 @@ checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 dependencies = [
  "cfg-if 0.1.10",
  "libc",
- "rand",
+ "rand 0.7.3",
  "redox_syscall",
  "remove_dir_all",
  "winapi 0.3.9",
@@ -2019,9 +2233,9 @@ version = "1.0.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -2065,10 +2279,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa"
 dependencies = [
  "proc-macro-hack",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
  "standback",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -2107,9 +2321,9 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -2170,9 +2384,9 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
 ]
 
 [[package]]
@@ -2233,6 +2447,12 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
 
+[[package]]
+name = "unicode-xid"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.1"
@@ -2262,7 +2482,7 @@ version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
 dependencies = [
- "rand",
+ "rand 0.7.3",
 ]
 
 [[package]]
@@ -2314,9 +2534,9 @@ dependencies = [
  "bumpalo",
  "lazy_static",
  "log",
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
  "wasm-bindgen-shared",
 ]
 
@@ -2338,7 +2558,7 @@ version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038"
 dependencies = [
- "quote",
+ "quote 1.0.7",
  "wasm-bindgen-macro-support",
 ]
 
@@ -2348,9 +2568,9 @@ version = "0.2.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -2449,8 +2669,8 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "syn",
+ "syn 1.0.48",
  "synstructure",
 ]
 
@@ -2469,8 +2689,8 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3f369ddb18862aba61aa49bf31e74d29f0f162dec753063200e1dc084345d16"
 dependencies = [
- "proc-macro2",
+ "proc-macro2 1.0.24",
- "quote",
+ "quote 1.0.7",
- "syn",
+ "syn 1.0.48",
  "synstructure",
 ]

Cargo.toml

@@ -27,6 +27,7 @@ sled = "0.34"
 zerocopy = "0.3"
 byteorder = "1.3"
 futures = "0.3"
+phf = { version = "0.7", features = ["macros"] }
 olm-sys = "1.0"
 matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk", rev = "master" }
 

src/bin/dicebot.rs

@@ -29,6 +29,8 @@ async fn run() -> Result<(), BotError> {
     let db = Database::new(&cfg.database_path())?;
     let state = Arc::new(RwLock::new(DiceBotState::new(&cfg)));
 
+    db.migrate(cfg.migration_version())?;
+
     match DiceBot::new(&cfg, &state, &db) {
         Ok(bot) => bot.run().await?,
         Err(e) => println!("Error connecting: {:?}", e),

src/config.rs

@@ -4,6 +4,10 @@ use std::fs;
 use std::path::PathBuf;
 use thiserror::Error;
 
+/// Shortcut to defining db migration versions. Will probably
+/// eventually be moved to a config file.
+const MIGRATION_VERSION: u32 = 1;
+
 #[derive(Error, Debug)]
 pub enum ConfigError {
     #[error("i/o error: {0}")]
@@ -124,6 +128,15 @@ impl Config {
             .unwrap_or_else(|| db_path_from_env())
     }
 
+    /// The current migration version we expect of the database. If
+    /// this number is higher than the one in the database, we will
+    /// execute migrations to update the data.
+    #[inline]
+    #[must_use]
+    pub fn migration_version(&self) -> u32 {
+        MIGRATION_VERSION
+    }
+
     /// Figure out the allowed oldest message age, in seconds. This will
     /// be the defined oldest message age in the bot config, if the bot
     /// configuration and associated "oldest_message_age" setting are

src/db.rs

@@ -1,9 +1,13 @@
-use crate::db::errors::DataError;
+use crate::db::errors::{DataError, MigrationError};
+use crate::db::migrations::{get_migration_version, Migrations};
 use crate::db::variables::Variables;
+use log::info;
 use sled::Db;
 use std::path::Path;
 
+pub mod data_migrations;
 pub mod errors;
+pub mod migrations;
 pub mod schema;
 pub mod variables;
 
@@ -11,19 +15,59 @@ pub mod variables;
 pub struct Database {
     db: Db,
     pub(crate) variables: Variables,
-    //rooms: Tree,
+    pub(crate) migrations: Migrations,
 }
 
 impl Database {
     pub fn new<P: AsRef<Path>>(path: P) -> Result<Database, DataError> {
         let db = sled::open(path)?;
         let variables = db.open_tree("variables")?;
-        //let rooms = db.open_tree("rooms")?;
+        let migrations = db.open_tree("migrations")?;
 
         Ok(Database {
             db: db.clone(),
             variables: Variables(variables),
-            //rooms: rooms,
+            migrations: Migrations(migrations),
         })
     }
+
+    pub fn migrate(&self, to_version: u32) -> Result<(), DataError> {
+        //get version from db
+        let db_version = get_migration_version(&self)?;
+
+        if db_version < to_version {
+            info!(
+                "Migrating database from version {} to version {}",
+                db_version, to_version
+            );
+            //if db version < to_version, proceed
+            //produce range of db_version+1 .. to_version (inclusive)
+            let versions_to_run: Vec<u32> = ((db_version + 1)..=to_version).collect();
+            let migrations = data_migrations::get_migrations(&versions_to_run)?;
+
+            //execute each closure.
+            for (version, migration_func) in versions_to_run.iter().zip(migrations) {
+                //This needs to be transactional on migrations
+                //keyspace. abort on migration func error.
+
+                info!("Applying migration: {}", version);
+                match migration_func(&self) {
+                    Ok(_) => Ok(()),
+                    Err(e) => Err(e),
+                }?;
+
+                self.migrations.set_migration_version(*version)?;
+            }
+
+            info!("Done applying migrations.");
+            Ok(())
+        } else if db_version > to_version {
+            //if db version > to_version, cannot downgrade error
+            Err(MigrationError::CannotDowngrade.into())
+        } else {
+            //if db version == to_version, do nothing
+            info!("No database migrations needed.");
+            Ok(())
+        }
+    }
 }

src/db/data_migrations.rs

@@ -0,0 +1,22 @@
+use crate::db::errors::{DataError, MigrationError};
+use crate::db::Database;
+use phf::phf_map;
+
+pub(super) type DataMigration = fn(&Database) -> Result<(), DataError>;
+
+static MIGRATIONS: phf::Map<u32, DataMigration> = phf_map! {
+    1u32 => super::variables::migrations::migration1,
+};
+
+pub fn get_migrations(versions: &[u32]) -> Result<Vec<DataMigration>, MigrationError> {
+    let mut migrations: Vec<DataMigration> = vec![];
+
+    for version in versions {
+        match MIGRATIONS.get(version) {
+            Some(func) => migrations.push(*func),
+            None => return Err(MigrationError::MigrationNotFound(*version)),
+        }
+    }
+
+    Ok(migrations)
+}

src/db/errors.rs

@@ -1,6 +1,18 @@
 use sled::transaction::{TransactionError, UnabortableTransactionError};
 use thiserror::Error;
 
+#[derive(Error, Debug)]
+pub enum MigrationError {
+    #[error("cannot downgrade to an older database version")]
+    CannotDowngrade,
+
+    #[error("migration for version {0} not defined")]
+    MigrationNotFound(u32),
+
+    #[error("migration failed: {0}")]
+    MigrationFailed(String),
+}
+
 //TODO better combining of key and value in certain errors (namely
 //I32SchemaViolation).
 #[derive(Error, Debug)]
@@ -22,6 +34,9 @@ pub enum DataError {
 
     #[error("unabortable transaction error: {0}")]
     UnabortableTransactionError(#[from] UnabortableTransactionError),
+
+    #[error("data migration error: {0}")]
+    MigrationError(#[from] MigrationError),
 }
 
 /// This From implementation is necessary to deal with the recursive
@@ -40,3 +55,14 @@ impl From<TransactionError<DataError>> for DataError {
         }
     }
 }
+
+// impl From<ConflictableTransactionError<DataError>> for DataError {
+//     fn from(error: ConflictableTransactionError<DataError>) -> Self {
+//         match error {
+//             ConflictableTransactionError::Abort(data_err) => data_err,
+//             ConflictableTransactionError::Storage(storage_err) => {
+//                 DataError::TransactionError(TransactionError::Storage(storage_err))
+//             }
+//         }
+//     }
+// }

src/db/migrations.rs

@@ -0,0 +1,54 @@
+use crate::db::errors::DataError;
+use crate::db::schema::convert_u32;
+use crate::db::Database;
+use byteorder::LittleEndian;
+use sled::Tree;
+use zerocopy::byteorder::U32;
+use zerocopy::AsBytes;
+
+//This file is for controlling the migration info stored in the
+//database, not actually running migrations.
+
+#[derive(Clone)]
+pub struct Migrations(pub(super) Tree);
+
+const COLON: &'static [u8] = b":";
+const METADATA_SPACE: &'static str = "metadata";
+const MIGRATION_KEY: &'static str = "migration_version";
+
+fn to_key(keyspace: &str, key_name: &str) -> Vec<u8> {
+    let mut key = vec![];
+    key.extend_from_slice(keyspace.as_bytes());
+    key.extend_from_slice(COLON);
+    key.extend_from_slice(key_name.as_bytes());
+    key
+}
+
+fn metadata_key(key_name: &str) -> Vec<u8> {
+    to_key(METADATA_SPACE, key_name)
+}
+
+impl Migrations {
+    pub(super) fn set_migration_version(&self, version: u32) -> Result<(), DataError> {
+        //Rust cannot type infer this transaction
+        let result: Result<_, sled::transaction::TransactionError<DataError>> =
+            self.0.transaction(|tx| {
+                let key = metadata_key(MIGRATION_KEY);
+                let db_value: U32<LittleEndian> = U32::new(version);
+                tx.insert(key, db_value.as_bytes())?;
+                Ok(())
+            });
+
+        result?;
+
+        Ok(())
+    }
+}
+
+pub(super) fn get_migration_version(db: &Database) -> Result<u32, DataError> {
+    let key = metadata_key(MIGRATION_KEY);
+    match db.migrations.0.get(key)? {
+        Some(bytes) => convert_u32(&bytes),
+        None => Ok(0),
+    }
+}

src/db/schema.rs

@@ -1,6 +1,6 @@
 use crate::db::errors::DataError;
 use byteorder::LittleEndian;
-use zerocopy::byteorder::I32;
+use zerocopy::byteorder::{I32, U32};
 use zerocopy::LayoutVerified;
 
 /// User variables are stored as little-endian 32-bit integers in the
@@ -8,6 +8,8 @@ use zerocopy::LayoutVerified;
 /// read.
 type LittleEndianI32Layout<'a> = LayoutVerified<&'a [u8], I32<LittleEndian>>;
 
+type LittleEndianU32Layout<'a> = LayoutVerified<&'a [u8], U32<LittleEndian>>;
+
 /// Convert bytes to an i32 with zero-copy deserialization. An error
 /// is returned if the bytes do not represent an i32.
 pub(super) fn convert_i32(raw_value: &[u8]) -> Result<i32, DataError> {
@@ -20,3 +22,14 @@ pub(super) fn convert_i32(raw_value: &[u8]) -> Result<i32, DataError> {
         Err(DataError::I32SchemaViolation)
     }
 }
+
+pub(super) fn convert_u32(raw_value: &[u8]) -> Result<u32, DataError> {
+    let layout = LittleEndianU32Layout::new_unaligned(raw_value.as_ref());
+
+    if let Some(layout) = layout {
+        let value: U32<LittleEndian> = *layout;
+        Ok(value.get())
+    } else {
+        Err(DataError::I32SchemaViolation)
+    }
+}

src/db/variables.rs

@@ -1,46 +1,71 @@
 use crate::db::errors::DataError;
 use crate::db::schema::convert_i32;
 use byteorder::LittleEndian;
-use sled::transaction::abort;
+use sled::transaction::{abort, TransactionalTree};
-use sled::transaction::TransactionalTree;
 use sled::Tree;
 use std::collections::HashMap;
+use std::convert::From;
 use std::str;
 use zerocopy::byteorder::I32;
 use zerocopy::AsBytes;
 
-const METADATA_KEY: &'static str = "metadata";
+pub(super) mod migrations;
+
+const METADATA_SPACE: &'static [u8] = b"metadata";
+const VARIABLE_SPACE: &'static [u8] = b"variables";
+
 const VARIABLE_COUNT_KEY: &'static str = "variable_count";
 
 #[derive(Clone)]
-pub struct Variables(pub(crate) Tree);
+pub struct Variables(pub(super) Tree);
 
-fn to_key(room_id: &str, username: &str, variable_name: &str) -> Vec<u8> {
+//TODO at least some of these will probalby move elsewhere.
-    let mut key = vec![];
+
-    key.extend_from_slice(room_id.as_bytes());
+fn space_prefix<D: Into<Vec<u8>>>(space: &[u8], delineator: D) -> Vec<u8> {
-    key.extend_from_slice(username.as_bytes());
+    let mut metadata_prefix = vec![];
-    key.extend_from_slice(variable_name.as_bytes());
+    metadata_prefix.extend_from_slice(space);
-    key
+    metadata_prefix.push(0xff);
+    let delineator = delineator.into();
+
+    if delineator.len() > 0 {
+        metadata_prefix.extend_from_slice(delineator.as_bytes());
+        metadata_prefix.push(0xff);
+    }
+
+    metadata_prefix
 }
 
-fn metadata_key(room_id: &str, username: &str, metadata_key: &str) -> Vec<u8> {
+fn metadata_space_prefix<D: Into<Vec<u8>>>(delineator: D) -> Vec<u8> {
-    let mut key = vec![];
+    space_prefix(METADATA_SPACE, delineator)
-    key.extend_from_slice(room_id.as_bytes());
-    key.extend_from_slice(METADATA_KEY.as_bytes());
-    key.extend_from_slice(username.as_bytes());
-    key.extend_from_slice(metadata_key.as_bytes());
-    key
 }
 
-fn room_variable_count_key(room_id: &str, username: &str) -> Vec<u8> {
+fn metadata_space_key<D: Into<Vec<u8>>>(delineator: D, key_name: &str) -> Vec<u8> {
-    metadata_key(room_id, username, VARIABLE_COUNT_KEY)
+    let mut metadata_key = metadata_space_prefix(delineator);
+    metadata_key.extend_from_slice(key_name.as_bytes());
+    metadata_key
 }
 
-fn to_prefix(room_id: &str, username: &str) -> Vec<u8> {
+fn variables_space_prefix<D: Into<Vec<u8>>>(delineator: D) -> Vec<u8> {
-    let mut prefix = vec![];
+    space_prefix(VARIABLE_SPACE, delineator)
-    prefix.extend_from_slice(room_id.as_bytes());
+}
-    prefix.extend_from_slice(username.as_bytes());
+
-    prefix
+fn variables_space_key<D: Into<Vec<u8>>>(delineator: D, key_name: &str) -> Vec<u8> {
+    let mut metadata_key = variables_space_prefix(delineator);
+    metadata_key.extend_from_slice(key_name.as_bytes());
+    metadata_key
+}
+
+/// Delineator for keeping track of a key by room ID and username.
+struct RoomAndUser<'a>(&'a str, &'a str);
+
+impl<'a> From<RoomAndUser<'a>> for Vec<u8> {
+    fn from(value: RoomAndUser<'a>) -> Vec<u8> {
+        let mut bytes = vec![];
+        bytes.extend_from_slice(value.0.as_bytes());
+        bytes.push(0xff);
+        bytes.extend_from_slice(value.1.as_bytes());
+        bytes
+    }
 }
 
 /// Use a transaction to atomically alter the count of variables in
@@ -51,7 +76,8 @@ fn alter_room_variable_count(
     username: &str,
     amount: i32,
 ) -> Result<i32, DataError> {
-    let key = room_variable_count_key(room_id, username);
+    let key = metadata_space_key(RoomAndUser(room_id, username), VARIABLE_COUNT_KEY);
+
     let mut new_count = match variables.get(&key)? {
         Some(bytes) => convert_i32(&bytes)? + amount,
         None => amount,
@@ -72,7 +98,7 @@ impl Variables {
         room_id: &str,
         username: &str,
     ) -> Result<HashMap<String, i32>, DataError> {
-        let prefix = to_prefix(&room_id, &username);
+        let prefix = variables_space_prefix(RoomAndUser(room_id, username));
         let prefix_len: usize = prefix.len();
 
         let variables: Result<Vec<_>, DataError> = self
@@ -94,7 +120,9 @@ impl Variables {
     }
 
     pub fn get_variable_count(&self, room_id: &str, username: &str) -> Result<i32, DataError> {
-        let key = room_variable_count_key(room_id, username);
+        let delineator = RoomAndUser(room_id, username);
+        let key = metadata_space_key(delineator, VARIABLE_COUNT_KEY);
+
         if let Some(raw_value) = self.0.get(&key)? {
             convert_i32(&raw_value)
         } else {
@@ -108,12 +136,12 @@ impl Variables {
         username: &str,
         variable_name: &str,
     ) -> Result<i32, DataError> {
-        let key = to_key(room_id, username, variable_name);
+        let key = variables_space_key(RoomAndUser(room_id, username), variable_name);
 
         if let Some(raw_value) = self.0.get(&key)? {
             convert_i32(&raw_value)
         } else {
-            Err(DataError::KeyDoesNotExist(String::from_utf8(key).unwrap()))
+            Err(DataError::KeyDoesNotExist(variable_name.to_owned()))
         }
     }
 
@@ -126,7 +154,7 @@ impl Variables {
     ) -> Result<(), DataError> {
         self.0
             .transaction(|tx| {
-                let key = to_key(room_id, username, variable_name);
+                let key = variables_space_key(RoomAndUser(room_id, username), variable_name);
                 let db_value: I32<LittleEndian> = I32::new(value);
                 let old_value = tx.insert(key, db_value.as_bytes())?;
 
@@ -151,14 +179,16 @@ impl Variables {
     ) -> Result<(), DataError> {
         self.0
             .transaction(|tx| {
-                let key = to_key(room_id, username, variable_name);
+                let key = variables_space_key(RoomAndUser(room_id, username), variable_name);
+
+                //TODO why does tx.remove require moving the key?
                 if let Some(_) = tx.remove(key.clone())? {
                     match alter_room_variable_count(&tx, room_id, username, -1) {
                         Err(e) => abort(e),
                         _ => Ok(()),
                     }
                 } else {
-                    abort(DataError::KeyDoesNotExist(String::from_utf8(key).unwrap()))
+                    abort(DataError::KeyDoesNotExist(variable_name.to_owned()))
                 }
             })
             .map_err(|e| e.into())

src/db/variables/migrations.rs

@@ -0,0 +1,68 @@
+use super::*;
+use crate::db::errors::{DataError, MigrationError};
+use crate::db::Database;
+use byteorder::LittleEndian;
+use sled::transaction::TransactionError;
+use sled::Batch;
+use zerocopy::byteorder::U32;
+use zerocopy::AsBytes;
+
+//TODO we will make this set variable count properly.
+pub(in crate::db) fn migration1(db: &Database) -> Result<(), DataError> {
+    let tree = &db.variables.0;
+    let prefix = variables_space_prefix("");
+
+    //Extract a vec of tuples, consisting of room id + username.
+    let results: Vec<(String, String)> = tree
+        .scan_prefix(prefix)
+        .map(|entry| {
+            if let Ok((key, _)) = entry {
+                let keys: Vec<Result<&str, _>> = key
+                    .split(|&b| b == 0xff)
+                    .map(|b| str::from_utf8(b))
+                    .collect();
+
+                if let &[_, Ok(room_id), Ok(username), Ok(_variable)] = keys.as_slice() {
+                    Ok((room_id.to_owned(), username.to_owned()))
+                } else {
+                    Err(MigrationError::MigrationFailed(
+                        "a key violates utf8 schema".to_string(),
+                    ))
+                }
+            } else {
+                Err(MigrationError::MigrationFailed(
+                    "encountered unexpected key".to_string(),
+                ))
+            }
+        })
+        .collect::<Result<Vec<_>, MigrationError>>()?;
+
+    let counts: HashMap<(String, String), u32> =
+        results
+            .into_iter()
+            .fold(HashMap::new(), |mut count_map, room_and_user| {
+                let count = count_map.entry(room_and_user).or_insert(0);
+                *count += 1;
+                count_map
+            });
+
+    //Start a transaction on the variables tree.
+    //Delete the old variable_count variable if exists.
+    //Add variable count according to new schema.
+    let tx_result: Result<_, TransactionError<DataError>> = db.variables.0.transaction(|tx_vars| {
+        let batch = counts.iter().fold(Batch::default(), |mut batch, entry| {
+            let key =
+                variables_space_key(RoomAndUser(&(entry.0).0, &(entry.0).1), VARIABLE_COUNT_KEY);
+
+            let db_value: U32<LittleEndian> = U32::new(*entry.1);
+            batch.insert(key, db_value.as_bytes());
+            batch
+        });
+
+        tx_vars.apply_batch(&batch)?;
+        Ok(())
+    });
+
+    tx_result?; //For some reason, it cannot infer the type
+    Ok(())
+}