Compare commits
974
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32ca2e8a73 | ||
|
|
d7f6bc1614 | ||
|
|
83d878f111 | ||
|
|
0fc6441c96 | ||
|
|
5be1620629 | ||
|
|
685b87e2ef | ||
|
|
72429c2106 | ||
|
|
7707fe15a9 | ||
|
|
c583ffff41 | ||
|
|
7c1ab8b8e2 | ||
|
|
570fbd0149 | ||
|
|
da1f09e9b5 | ||
|
|
b2fb4f4b9b | ||
|
|
51277b17a7 | ||
|
|
32f5ef864e | ||
|
|
a8f28297ec | ||
|
|
0ae13c9731 | ||
|
|
1895dbb768 | ||
|
|
a6cbbd0f31 | ||
|
|
f1f52af9cf | ||
|
|
e3c619eca4 | ||
|
|
57e59d97db | ||
|
|
d7cada0f38 | ||
|
|
fd2ac1fcfc | ||
|
|
0181b0e7a1 | ||
|
|
2b5e5cfa4c | ||
|
|
bfb0d02622 | ||
|
|
534b70643b | ||
|
|
3d0f729572 | ||
|
|
fd4ed1e425 | ||
|
|
693450393c | ||
|
|
8ee7584cff | ||
|
|
0a56a73bd9 | ||
|
|
3a1d3090f2 | ||
|
|
0727393627 | ||
|
|
d2ef059471 | ||
|
|
2e894a512b | ||
|
|
34b0209f05 | ||
|
|
046f575632 | ||
|
|
1c77104b5e | ||
|
|
386be8990a | ||
|
|
76d9c7c82a | ||
|
|
98f08034a1 | ||
|
|
308cf77437 | ||
|
|
ce64f469f4 | ||
|
|
e632c3db1b | ||
|
|
239d22ed72 | ||
|
|
fefbe96816 | ||
|
|
b1f662d639 | ||
|
|
aa5b27b040 | ||
|
|
10d254ec1b | ||
|
|
e8a44821d0 | ||
|
|
92e48f0717 | ||
|
|
9742c44db5 | ||
|
|
5d16f51950 | ||
|
|
1ed2e8b5bc | ||
|
|
b20803b766 | ||
|
|
8cf1daf823 | ||
|
|
083b54472c | ||
|
|
1b9fea0669 | ||
|
|
d3c37c7159 | ||
|
|
807f020184 | ||
|
|
b400be68b7 | ||
|
|
e39bdb58ec | ||
|
|
83c8977033 | ||
|
|
682990eeb3 | ||
|
|
7ee766aee1 | ||
|
|
848c5ad784 | ||
|
|
a41fb1b036 | ||
|
|
915117ae7b | ||
|
|
de5fa78762 | ||
|
|
b0c8efd840 | ||
|
|
1dcd5edbf9 | ||
|
|
87d7fc8e5d | ||
|
|
100a747eab | ||
|
|
105fd03ef8 | ||
|
|
9395a9c1d9 | ||
|
|
02f7efd9d4 | ||
|
|
aea86c5977 | ||
|
|
93ebb31b17 | ||
|
|
bfed57e3e0 | ||
|
|
23647390e6 | ||
|
|
96bca72bb4 | ||
|
|
6163e7b13a | ||
|
|
66547bd28a | ||
|
|
e6ab823473 | ||
|
|
a2c0bce4cc | ||
|
|
2c8239f9cb | ||
|
|
e9551c9e8d | ||
|
|
0ac80086bc | ||
|
|
9ab36ddc36 | ||
|
|
a6bc34c09b | ||
|
|
3bb6b16c4a | ||
|
|
1cde2442aa | ||
|
|
2910cee32b | ||
|
|
7fefc7b245 | ||
|
|
68d213f4c9 | ||
|
|
7d9a2ce827 | ||
|
|
c6ef287452 | ||
|
|
e6091a2b7e | ||
|
|
dfea168499 | ||
|
|
241d1bd7e1 | ||
|
|
56addcd172 | ||
|
|
8d8e1810d4 | ||
|
|
b5ddc11c46 | ||
|
|
f107746e90 | ||
|
|
f68a26381a | ||
|
|
5a2cc90786 | ||
|
|
3881d5c1f2 | ||
|
|
958f2c00c8 | ||
|
|
165333b0fc | ||
|
|
cc827e157a | ||
|
|
7b4e25008a | ||
|
|
f66e5a9c3d | ||
|
|
9954f5b4a4 | ||
|
|
53936661c8 | ||
|
|
e2552ff8a9 | ||
|
|
6ab08cc3e1 | ||
|
|
3205373bb3 | ||
|
|
3b8d1f7076 | ||
|
|
5832c25ad1 | ||
|
|
569a3d9ff8 | ||
|
|
69bc7108bd | ||
|
|
8a557e9d71 | ||
|
|
fe0f071993 | ||
|
|
effa53339c | ||
|
|
f6fb0e4620 | ||
|
|
cf46a014c1 | ||
|
|
d72b5fa16d | ||
|
|
0151507cef | ||
|
|
0fea781f8f | ||
|
|
3c4cadc020 | ||
|
|
e2615ccf88 | ||
|
|
595deb0e13 | ||
|
|
5372d1aab2 | ||
|
|
52eca14ca6 | ||
|
|
1155f5ac63 | ||
|
|
7319953ae3 | ||
|
|
b8e106a72e | ||
|
|
f30f8eba2f | ||
|
|
7ed2296fc5 | ||
|
|
0691a46669 | ||
|
|
d7f01c1f7d | ||
|
|
4d71929c60 | ||
|
|
dadce055f8 | ||
|
|
d2867580ab | ||
|
|
8dc238efae | ||
|
|
7cd4b5e11c | ||
|
|
00043f9509 | ||
|
|
9b81f3f263 | ||
|
|
2747d1a1a9 | ||
|
|
9f7ed0b232 | ||
|
|
c176ea7366 | ||
|
|
d50e4bb819 | ||
|
|
4485ca74f6 | ||
|
|
7f72b214e5 | ||
|
|
bc8cc1adb3 | ||
|
|
d03c2de901 | ||
|
|
7d3c5f1e64 | ||
|
|
81573de393 | ||
|
|
a80dd5d001 | ||
|
|
7e0077f293 | ||
|
|
1eb035e6ad | ||
|
|
bdd14ac517 | ||
|
|
63e5d00bdf | ||
|
|
c25fb3d74f | ||
|
|
572d41325b | ||
|
|
e02120cdf7 | ||
|
|
7b1221a74d | ||
|
|
d6b4f994ad | ||
|
|
66a41dafc4 | ||
|
|
f19b9c155e | ||
|
|
5d016fdb6b | ||
|
|
cae9924fd9 | ||
|
|
089f28e8ba | ||
|
|
dcb44d3a09 | ||
|
|
f99cba0b92 | ||
|
|
e10dc0dace | ||
|
|
49a89d636b | ||
|
|
b94f91b9b2 | ||
|
|
06a17c8511 | ||
|
|
eebd70aaa6 | ||
|
|
779c775df9 | ||
|
|
1d4d2925c2 | ||
|
|
b67db7c2d5 | ||
|
|
18953b2998 | ||
|
|
527b7eb074 | ||
|
|
34a97c447f | ||
|
|
1c525e20b1 | ||
|
|
0bcce50285 | ||
|
|
96e0b2a2b1 | ||
|
|
f3794d2c42 | ||
|
|
97fd736923 | ||
|
|
ba950a3d1f | ||
|
|
dad5a902d3 | ||
|
|
3e33fbcd35 | ||
|
|
444555d3e4 | ||
|
|
6a74f524e5 | ||
|
|
9c5b59ac01 | ||
|
|
1840601f8e | ||
|
|
88962af8b2 | ||
|
|
df22cdf6e9 | ||
|
|
041446005b | ||
|
|
7c39a9b0e1 | ||
|
|
05cd92f766 | ||
|
|
d724e7226c | ||
|
|
738c782f02 | ||
|
|
33b05c859a | ||
|
|
ed369d7223 | ||
|
|
fb32bb12b1 | ||
|
|
3e996035ea | ||
|
|
ae8a94d615 | ||
|
|
69a74a567b | ||
|
|
1b46d4f09a | ||
|
|
c6f13892e0 | ||
|
|
05e0f32345 | ||
|
|
8171e6de13 | ||
|
|
b042495f37 | ||
|
|
420ccdfbae | ||
|
|
41466a4147 | ||
|
|
9798f2d711 | ||
|
|
5f3543921b | ||
|
|
a41366cc3a | ||
|
|
4c7e4b91a6 | ||
|
|
28183f4b66 | ||
|
|
a0abc1eddd | ||
|
|
676644f2db | ||
|
|
22ac56d62d | ||
|
|
143f7f634d | ||
|
|
955faef23a | ||
|
|
2e159e6ea3 | ||
|
|
02b533fec3 | ||
|
|
de3099e6d8 | ||
|
|
368f67f422 | ||
|
|
13c9a094d1 | ||
|
|
0f8b7a19ef | ||
|
|
8f7d977f90 | ||
|
|
064440ee84 | ||
|
|
50641d0569 | ||
|
|
f5ecc77e29 | ||
|
|
655bb6e21c | ||
|
|
ab4cdd2ed4 | ||
|
|
d3834c5069 | ||
|
|
7a1b2c58c3 | ||
|
|
7fc6d6c6b1 | ||
|
|
dec5db4e71 | ||
|
|
d47aeb1fc8 | ||
|
|
e47584b286 | ||
|
|
f17d178d03 | ||
|
|
d88fd1588d | ||
|
|
119797014c | ||
|
|
e8f568644b | ||
|
|
b736a95b0d | ||
|
|
d909d12cb3 | ||
|
|
201ffa9d62 | ||
|
|
5619f51528 | ||
|
|
d99d06b125 | ||
|
|
1b9ade7f65 | ||
|
|
e01312642b | ||
|
|
cd9514f4c8 | ||
|
|
f111c23415 | ||
|
|
6857c4c208 | ||
|
|
52897f8f9a | ||
|
|
8bc70e9840 | ||
|
|
5b81303f04 | ||
|
|
ede524c064 | ||
|
|
aeed09c346 | ||
|
|
70b80aa5fa | ||
|
|
e8d4d095cc | ||
|
|
c50a3568a5 | ||
|
|
17e96b91b5 | ||
|
|
c5e7a8517b | ||
|
|
c36773777d | ||
|
|
d69e0b8561 | ||
|
|
19ccc22a43 | ||
|
|
6968a0cfcb | ||
|
|
df618fbd9f | ||
|
|
94acd43340 | ||
|
|
14bdd745d1 | ||
|
|
aa3f5de16f | ||
|
|
9f7890cb1a | ||
|
|
1dc34c16b5 | ||
|
|
b0e7be7ce2 | ||
|
|
026ab4c922 | ||
|
|
07ba64c200 | ||
|
|
b4b35cf904 | ||
|
|
ba974c9810 | ||
|
|
9817db7edd | ||
|
|
515bb0e6bd | ||
|
|
bc7ea8fb20 | ||
|
|
2caf8e4c80 | ||
|
|
e1f3cd6ec3 | ||
|
|
02e821530a | ||
|
|
b6c3d80edb | ||
|
|
0042586190 | ||
|
|
c6dce838b7 | ||
|
|
5b885bb164 | ||
|
|
95f46e6077 | ||
|
|
31f58311e3 | ||
|
|
130613e166 | ||
|
|
50fd4fa7e6 | ||
|
|
e6a9a6b428 | ||
|
|
d6be93537e | ||
|
|
1ef35cf466 | ||
|
|
1eb0e3c1c6 | ||
|
|
e50313b207 | ||
|
|
ac3f8d1ac7 | ||
|
|
202b97eff7 | ||
|
|
f25aebf269 | ||
|
|
9b1e636a66 | ||
|
|
5147f0153a | ||
|
|
88f7739ae0 | ||
|
|
c0c033a0c1 | ||
|
|
73300c5700 | ||
|
|
37033179fe | ||
|
|
3e6a5ac9cf | ||
|
|
dcd297eb5b | ||
|
|
4a9742e696 | ||
|
|
8edbae2e04 | ||
|
|
63c6735ffc | ||
|
|
8a0441c144 | ||
|
|
460c5ed2b3 | ||
|
|
8f44f0937b | ||
|
|
1803b1587b | ||
|
|
a5325061f2 | ||
|
|
a10a0df09f | ||
|
|
031044c7ea | ||
|
|
ca755e4905 | ||
|
|
3063a028b3 | ||
|
|
2c659acc7d | ||
|
|
341a8caef9 | ||
|
|
366f8c2dc6 | ||
|
|
5803540422 | ||
|
|
f17c492a63 | ||
|
|
aad6a7ce15 | ||
|
|
1cb48c7bf6 | ||
|
|
f25c405ad2 | ||
|
|
c9920849dd | ||
|
|
81b240f81d | ||
|
|
6a902b29e1 | ||
|
|
9c31ac7925 | ||
|
|
f0d8d1680f | ||
|
|
6dcbcacb10 | ||
|
|
3196717cee | ||
|
|
f95578ac02 | ||
|
|
57ca6facc3 | ||
|
|
561a1f8d8a | ||
|
|
1449c8b9fe | ||
|
|
1938af27f5 | ||
|
|
123bc125cf | ||
|
|
ecbe168b54 | ||
|
|
00cef78858 | ||
|
|
d05da0fabd | ||
|
|
4db3f2a443 | ||
|
|
fd710d590a | ||
|
|
9b98c9edd3 | ||
|
|
4893eedacb | ||
|
|
fb50163b47 | ||
|
|
3b8463f9d2 | ||
|
|
42c7677690 | ||
|
|
8665f25031 | ||
|
|
1c6b573147 | ||
|
|
90162da1a6 | ||
|
|
1ca6c1dd56 | ||
|
|
2d27c8ecdb | ||
|
|
59cda9cfb9 | ||
|
|
091af64003 | ||
|
|
a12210c875 | ||
|
|
43649fc5b4 | ||
|
|
01cc25653a | ||
|
|
a010e4a4b2 | ||
|
|
817a11ff11 | ||
|
|
c52e1fc78c | ||
|
|
d74871da40 | ||
|
|
a25cf15ecd | ||
|
|
c505df8830 | ||
|
|
f214ada545 | ||
|
|
c10aa3b8b0 | ||
|
|
6311d709c3 | ||
|
|
8ab77465db | ||
|
|
201bf73ef7 | ||
|
|
49784c8122 | ||
|
|
3e03f308b5 | ||
|
|
95cecb0229 | ||
|
|
7676f5b15f | ||
|
|
d5b057534a | ||
|
|
7a15f6dfc7 | ||
|
|
712d825c23 | ||
|
|
4073758874 | ||
|
|
3543a3c763 | ||
|
|
3122912920 | ||
|
|
f11fdf6717 | ||
|
|
572bcca9c2 | ||
|
|
10fa064109 | ||
|
|
7cd779d304 | ||
|
|
628d168049 | ||
|
|
37423757b5 | ||
|
|
58752bb8fe | ||
|
|
7e7f70ff3e | ||
|
|
f77c227b2d | ||
|
|
21d842b0b0 | ||
|
|
86f9ed6248 | ||
|
|
dcd7b684bb | ||
|
|
f715d0b860 | ||
|
|
98d690b849 | ||
|
|
45535e8a63 | ||
|
|
c78895aaea | ||
|
|
d5ede50b04 | ||
|
|
fab6132cb3 | ||
|
|
e0a8709c09 | ||
|
|
17795fd5ce | ||
|
|
d6738ffa5d | ||
|
|
e72642667f | ||
|
|
3785b1efd4 | ||
|
|
7aeae62f84 | ||
|
|
e5e33f8cb7 | ||
|
|
a836d5947d | ||
|
|
7cdd5d9e41 | ||
|
|
3f5d3fec36 | ||
|
|
90094a2dca | ||
|
|
cc9924ffa6 | ||
|
|
5e6b4ab99e | ||
|
|
2d6d374162 | ||
|
|
3137d4f35c | ||
|
|
6ff9d23fd4 | ||
|
|
d4939c838b | ||
|
|
cd306f1d23 | ||
|
|
5fd07e0889 | ||
|
|
b0796ff5fd | ||
|
|
9dfa074cd6 | ||
|
|
033761c5f9 | ||
|
|
2926ac494e | ||
|
|
06143c2e8c | ||
|
|
f00d28fdab | ||
|
|
4705f29e2f | ||
|
|
b2cb10de5f | ||
|
|
1005a725d4 | ||
|
|
50768cd242 | ||
|
|
cc3ebbe519 | ||
|
|
8056e349e0 | ||
|
|
0001f59c8d | ||
|
|
2941792a85 | ||
|
|
320bc3c032 | ||
|
|
2825e51550 | ||
|
|
cbdd47d056 | ||
|
|
516c8c6e23 | ||
|
|
ef1236b600 | ||
|
|
28cbed1d13 | ||
|
|
cde199a797 | ||
|
|
5589a0a69c | ||
|
|
09e8fb17cd | ||
|
|
d71c4c2215 | ||
|
|
6f621e6480 | ||
|
|
8e33c2d301 | ||
|
|
52aca124c3 | ||
|
|
c7159d58f1 | ||
|
|
5d6e8fe3e3 | ||
|
|
99762dac23 | ||
|
|
57956b3fd2 | ||
|
|
b323645227 | ||
|
|
dd9502f373 | ||
|
|
aa8a3077a2 | ||
|
|
8e81e5f29e | ||
|
|
63dd1fb599 | ||
|
|
32561a9912 | ||
|
|
3fdc8608f3 | ||
|
|
9e2e2067c6 | ||
|
|
f112c77fcc | ||
|
|
888d69a87a | ||
|
|
bd183cdee1 | ||
|
|
95b4f899c5 | ||
|
|
2946155313 | ||
|
|
f2ca2aa1f7 | ||
|
|
b20bf10fda | ||
|
|
208c15bc16 | ||
|
|
c3d3c82f3a | ||
|
|
b22d9cc27d | ||
|
|
908cf5042e | ||
|
|
6ec1641294 | ||
|
|
7b79ec360b | ||
|
|
718aee242d | ||
|
|
56fc6b7deb | ||
|
|
528a43bd6c | ||
|
|
b357803362 | ||
|
|
448808529a | ||
|
|
a0bc208412 | ||
|
|
6b0172b9bf | ||
|
|
bc1eccb66d | ||
|
|
a9249d4a82 | ||
|
|
3c373494d6 | ||
|
|
40761e4031 | ||
|
|
5aef606d3b | ||
|
|
4671cd6002 | ||
|
|
2ec0b2cd20 | ||
|
|
07effa3925 | ||
|
|
cd76a71de3 | ||
|
|
5e63c0955d | ||
|
|
e523afa741 | ||
|
|
c2a98a60f9 | ||
|
|
6bc92d6d31 | ||
|
|
13cf8bbeb7 | ||
|
|
60c0d4104d | ||
|
|
52fd4445a4 | ||
|
|
693ba4ed4a | ||
|
|
5397f0b033 | ||
|
|
6c78ec7c41 | ||
|
|
fed4bd5269 | ||
|
|
c0882864f2 | ||
|
|
1a7e2e663e | ||
|
|
faa87ed839 | ||
|
|
83f585e30a | ||
|
|
7a244bb847 | ||
|
|
1b925119b5 | ||
|
|
3eb847e28f | ||
|
|
9dc8f30e47 | ||
|
|
3ce6441a0c | ||
|
|
6d00a1a5cf | ||
|
|
aaca32a5cd | ||
|
|
54c96d1995 | ||
|
|
e5a2d59446 | ||
|
|
d95bead383 | ||
|
|
51ea3cf30c | ||
|
|
6ad6c1d780 | ||
|
|
32f34f63fd | ||
|
|
98aefac09b | ||
|
|
5dc84850ad | ||
|
|
45b711f9d1 | ||
|
|
1ef187533c | ||
|
|
6a71de7835 | ||
|
|
18496b6285 | ||
|
|
b9db899f6c | ||
|
|
4e9d2f08b1 | ||
|
|
06ad1c15d7 | ||
|
|
30a603ce05 | ||
|
|
63ae679fe1 | ||
|
|
b92bbf4de8 | ||
|
|
4781eda722 | ||
|
|
2fd93c5789 | ||
|
|
329519d4aa | ||
|
|
52434a583a | ||
|
|
18a5f46b61 | ||
|
|
16a019dba0 | ||
|
|
1a4c3224c9 | ||
|
|
c3317686f8 | ||
|
|
4622472123 | ||
|
|
f1748f8c76 | ||
|
|
0c101dccc8 | ||
|
|
dc78bf1ccb | ||
|
|
a7cc3aac28 | ||
|
|
339631530a | ||
|
|
fb2bdf7d05 | ||
|
|
c45f1328ac | ||
|
|
050d58007f | ||
|
|
49d86746ed | ||
|
|
ef4e11ecb8 | ||
|
|
fd702e43f1 | ||
|
|
249e0fdd36 | ||
|
|
6208118b13 | ||
|
|
73020b6a93 | ||
|
|
a9667e84ad | ||
|
|
ab1524ff1a | ||
|
|
1069d0b4bf | ||
|
|
4a980f4ddb | ||
|
|
9fbb9a551e | ||
|
|
c24a7c03c2 | ||
|
|
5a8280d577 | ||
|
|
da83e84125 | ||
|
|
ef9a3cadcf | ||
|
|
c983d57f73 | ||
|
|
3204a39ada | ||
|
|
f4319c1280 | ||
|
|
762396d997 | ||
|
|
c575e7df94 | ||
|
|
aa6409f2e8 | ||
|
|
e603e57588 | ||
|
|
049b48ef79 | ||
|
|
0ba0a6ab0a | ||
|
|
7475ea1b0b | ||
|
|
d0654af74f | ||
|
|
d5f8df5997 | ||
|
|
39acb312ef | ||
|
|
b643c38204 | ||
|
|
dd70f5efd7 | ||
|
|
bb38628f0e | ||
|
|
0cf3af6857 | ||
|
|
edb993d52b | ||
|
|
aea333bb78 | ||
|
|
45817db8c3 | ||
|
|
dbef17c2e1 | ||
|
|
42a8f22c21 | ||
|
|
6e06e0dfc7 | ||
|
|
bf500ab44b | ||
|
|
4d4eca5433 | ||
|
|
4b01ad1e30 | ||
|
|
3b60929bdc | ||
|
|
a45d141597 | ||
|
|
8cbe254491 | ||
|
|
ab365ce3da | ||
|
|
aa3e70f9a0 | ||
|
|
cda8d42a76 | ||
|
|
67f32bf1b1 | ||
|
|
7a3f363f34 | ||
|
|
91eac2cdb3 | ||
|
|
98a928147f | ||
|
|
8a9045c028 | ||
|
|
6c65a11b8d | ||
|
|
7dd53182e0 | ||
|
|
53b95d830a | ||
|
|
3028a890ef | ||
|
|
39c6fbddb1 | ||
|
|
704cb4e929 | ||
|
|
8e4f323439 | ||
|
|
937702ea1d | ||
|
|
3ea56ffb38 | ||
|
|
becd3fc5a3 | ||
|
|
4c8603815c | ||
|
|
3ae75c2d2b | ||
|
|
9371c005aa | ||
|
|
16bd6b3f5d | ||
|
|
3dd78a6101 | ||
|
|
41b678814b | ||
|
|
d0ccb074d5 | ||
|
|
cb0fc7cca2 | ||
|
|
5e40ea6ee7 | ||
|
|
427c5999f1 | ||
|
|
a91f58b8fd | ||
|
|
3013799664 | ||
|
|
f8220a658e | ||
|
|
f1d2b29450 | ||
|
|
cb93752aff | ||
|
|
549cad378d | ||
|
|
631f863dd8 | ||
|
|
4396682c07 | ||
|
|
ca4ee4e3e4 | ||
|
|
2da09e0732 | ||
|
|
4986dc6da9 | ||
|
|
1a70af2ae1 | ||
|
|
5a8cb77d0d | ||
|
|
600fc241cf | ||
|
|
fa797408d9 | ||
|
|
3fac516950 | ||
|
|
2f65aa2337 | ||
|
|
f5210a8fb0 | ||
|
|
8f1295c00f | ||
|
|
18a3768e07 | ||
|
|
95e3dbc104 | ||
|
|
40355eac3e | ||
|
|
3a3b59a054 | ||
|
|
12e17cfd6e | ||
|
|
203e341bf4 | ||
|
|
6e6cd84ad7 | ||
|
|
252725d50f | ||
|
|
c15633ea9e | ||
|
|
27c9ae667e | ||
|
|
16f96efbe7 | ||
|
|
5787e7adc7 | ||
|
|
4631119179 | ||
|
|
0a0c2146c8 | ||
|
|
cedea7e400 | ||
|
|
66cef6646c | ||
|
|
64aa4b80e6 | ||
|
|
551a3cb1aa | ||
|
|
0942188b0a | ||
|
|
070c481871 | ||
|
|
6df4d29422 | ||
|
|
6e1e5bd08b | ||
|
|
ba365298ed | ||
|
|
e1c9620951 | ||
|
|
e776474f68 | ||
|
|
4589e861ab | ||
|
|
55324aa56a | ||
|
|
18d42a5c35 | ||
|
|
71d5a10621 | ||
|
|
ac595ebe37 | ||
|
|
50a6a3ca38 | ||
|
|
f83a89f82a | ||
|
|
bf905a78f0 | ||
|
|
fad9483fc9 | ||
|
|
46a78c69e7 | ||
|
|
33e0b3e1c0 | ||
|
|
4e9bfcf84b | ||
|
|
75c6b4c64a | ||
|
|
8536854266 | ||
|
|
6016dd0f8b | ||
|
|
47ef9e5cbd | ||
|
|
4858fd893c | ||
|
|
6b6e6f6c1e | ||
|
|
cbe5c04a0f | ||
|
|
b0a0f03b96 | ||
|
|
08b6a04055 | ||
|
|
f1982ca63d | ||
|
|
afe66288fe | ||
|
|
2794cfced4 | ||
|
|
d6b45a56e7 | ||
|
|
a55d980436 | ||
|
|
6037d1f326 | ||
|
|
0b43b51a2e | ||
|
|
184abac694 | ||
|
|
0ba5b2c3d6 | ||
|
|
0b3d74ea67 | ||
|
|
ea3bd614b8 | ||
|
|
d45edc493d | ||
|
|
271df3dad4 | ||
|
|
5a193872f8 | ||
|
|
b5512d77a1 | ||
|
|
42bda8eaba | ||
|
|
7e5d3f4f16 | ||
|
|
5f3920e491 | ||
|
|
3880d2616c | ||
|
|
156e596f9c | ||
|
|
fc58516cca | ||
|
|
a4efcd35af | ||
|
|
27fe1658a2 | ||
|
|
47feb2719c | ||
|
|
c21af6b351 | ||
|
|
821c12c482 | ||
|
|
4602f8c391 | ||
|
|
27a927fe00 | ||
|
|
3cdc6e8d84 | ||
|
|
89b52da387 | ||
|
|
50ad880ad5 | ||
|
|
15bed4bbcc | ||
|
|
10986ae3a8 | ||
|
|
2baa76b2ee | ||
|
|
ee460301fd | ||
|
|
8ebe2fce3d | ||
|
|
8198a697fd | ||
|
|
35acae360a | ||
|
|
8aaa8b0b11 | ||
|
|
25e18a7de9 | ||
|
|
bb090b6d20 | ||
|
|
a9c4e3970f | ||
|
|
d5ecc2347a | ||
|
|
58971a430d | ||
|
|
1512274f4c | ||
|
|
e310d33c12 | ||
|
|
ad32ba0343 | ||
|
|
4e30a5b4e5 | ||
|
|
c8516f3dc4 | ||
|
|
aae0fdc114 | ||
|
|
a19d0f67b9 | ||
|
|
971f69fe1f | ||
|
|
7468c691ad | ||
|
|
620c2ec141 | ||
|
|
3081b6f98b | ||
|
|
8630339e43 | ||
|
|
528d10091f | ||
|
|
149c3cf3dd | ||
|
|
f12c3cf9dd | ||
|
|
1f08a0efb3 | ||
|
|
6cf85d1d01 | ||
|
|
b5101a460d | ||
|
|
a5784ef09e | ||
|
|
122ec25bf0 | ||
|
|
d013ad9eba | ||
|
|
d918ca6fa5 | ||
|
|
852819180f | ||
|
|
218111eb72 | ||
|
|
1531cfb7ee | ||
|
|
16a89f21dc | ||
|
|
79e5160e37 | ||
|
|
510e966dc0 | ||
|
|
8238d3207e | ||
|
|
e37bc4ad40 | ||
|
|
0f78b8b917 | ||
|
|
19061fe04f | ||
|
|
9c901eb65f | ||
|
|
ae33ff3a6e | ||
|
|
59dbd625c1 | ||
|
|
8eb4e355b9 | ||
|
|
8b76b01bfe | ||
|
|
ec57b4b024 | ||
|
|
46d6caa47c | ||
|
|
97b65ce19c | ||
|
|
55d0ef40b0 | ||
|
|
a6c56416fc | ||
|
|
cd3c482d5e | ||
|
|
c5cbb77fc7 | ||
|
|
d7f21f02b3 | ||
|
|
87dc8ce9b3 | ||
|
|
2e22aa1a9c | ||
|
|
8c56dbf971 | ||
|
|
a78169e9e0 | ||
|
|
39a60c8120 | ||
|
|
c57d8760bd | ||
|
|
3eed2a8780 | ||
|
|
23b0268a7f | ||
|
|
3d0de946bb | ||
|
|
866b8ead91 | ||
|
|
aa274960ff | ||
|
|
54d6eb851b | ||
|
|
86d7276f24 | ||
|
|
0918651438 | ||
|
|
272d9297b6 | ||
|
|
f5bfc71899 | ||
|
|
42e1ded55f | ||
|
|
6c410af5a8 | ||
|
|
57ae676181 | ||
|
|
7896ca6e52 | ||
|
|
c4a315dd5e | ||
|
|
bf26959b72 | ||
|
|
37b2d046be | ||
|
|
fa2e407873 | ||
|
|
c61cf57db5 | ||
|
|
ef3690f2db | ||
|
|
14d01ec7bd | ||
|
|
5dc9f70386 | ||
|
|
88e593bf96 | ||
|
|
efde2e6be6 | ||
|
|
5a78a3bc46 | ||
|
|
b276d803d4 | ||
|
|
da51ac430b | ||
|
|
01eb0a3c7f | ||
|
|
dc2b3cf4f8 | ||
|
|
c4621a5ed2 | ||
|
|
1c953a4d5b | ||
|
|
c64ce7aa9c | ||
|
|
acdc4fdfec | ||
|
|
7ef9c18b72 | ||
|
|
a69b4e79db | ||
|
|
740fc6c2b8 | ||
|
|
42202c868d | ||
|
|
8c133b605c | ||
|
|
b50acd7238 | ||
|
|
54db278695 | ||
|
|
011948bcef | ||
|
|
9f95d8b38f | ||
|
|
3d394a6fa2 | ||
|
|
68d2b648e0 | ||
|
|
283998404f | ||
|
|
a7b4d0fea6 | ||
|
|
59051dd59d | ||
|
|
cd836a37e7 | ||
|
|
4326d22380 | ||
|
|
161d3a35ff | ||
|
|
20a9d21dfb | ||
|
|
e902789aa7 | ||
|
|
3745bbd95b | ||
|
|
e63b2880b0 | ||
|
|
a9b70e1ef7 | ||
|
|
e300f753c5 | ||
|
|
efaeca5887 | ||
|
|
4d67305661 | ||
|
|
853bb6ca3e | ||
|
|
b16f23ae8e | ||
|
|
91e6368de0 | ||
|
|
e4451b9628 | ||
|
|
770bb8ef5f | ||
|
|
87b958a210 | ||
|
|
314ac90ba9 | ||
|
|
9f33822083 | ||
|
|
10a4b35bc5 | ||
|
|
7b1dd0b60d | ||
|
|
7a650741f7 | ||
|
|
2ce5f1b944 | ||
|
|
063a14372d | ||
|
|
dd70e1b571 | ||
|
|
1840b893c2 | ||
|
|
8bcde757ce | ||
|
|
1dff223c71 | ||
|
|
a7aa2b0740 | ||
|
|
fb45fefbca | ||
|
|
7b914c2531 | ||
|
|
75d517a27f | ||
|
|
4c98bc690e | ||
|
|
550b3a3a99 | ||
|
|
ef7e89352d | ||
|
|
9322d604ee | ||
|
|
5d92110630 | ||
|
|
4056037cc2 | ||
|
|
677ae5487f | ||
|
|
19956b3807 | ||
|
|
ab9122f2cb | ||
|
|
e62c6b0ed0 | ||
|
|
27a420fbc2 | ||
|
|
b775ebb382 | ||
|
|
ac80a5c6a6 | ||
|
|
c74100a552 | ||
|
|
03379106d6 | ||
|
|
9ad229f663 | ||
|
|
0900edf776 | ||
|
|
ea520027b9 | ||
|
|
04679b3b3b | ||
|
|
2bf5c1324a | ||
|
|
3d93ca1bf6 | ||
|
|
c548431611 | ||
|
|
0ab29f7394 | ||
|
|
30dab7ad14 | ||
|
|
0c5fa59221 | ||
|
|
267e84f97d | ||
|
|
344cca75c6 | ||
|
|
0dd8e682f4 | ||
|
|
da7e54819a | ||
|
|
4e70215307 | ||
|
|
6aa162f6ff | ||
|
|
d380074569 | ||
|
|
c406e7ff51 | ||
|
|
31ff6cffee | ||
|
|
a9e525c81e | ||
|
|
6157a5fc1b | ||
|
|
be3cb1ad33 | ||
|
|
966f4a8144 | ||
|
|
261cf24efc | ||
|
|
a3da10f4eb | ||
|
|
6cb8063497 | ||
|
|
a45f212693 | ||
|
|
7c72cd5471 | ||
|
|
66f2032708 | ||
|
|
7546c7d8b4 | ||
|
|
703e4ad2fe | ||
|
|
0b98280a1f | ||
|
|
4609bed6a4 | ||
|
|
99ccb11d89 | ||
|
|
8a5a513662 | ||
|
|
bf0fd22309 | ||
|
|
cc98d10fc6 | ||
|
|
a6a022faa2 | ||
|
|
a864ad1c9c | ||
|
|
0bc586933f | ||
|
|
2785a8b4d4 | ||
|
|
d5d5f6ae06 | ||
|
|
be3ba1d53e | ||
|
|
09e6a4fee0 | ||
|
|
efc60dd8c5 | ||
|
|
a52cd974b9 | ||
|
|
de4217489c | ||
|
|
3c658b5012 | ||
|
|
f6814aa825 | ||
|
|
01a529cbda | ||
|
|
a1475af1e4 | ||
|
|
07594d4f95 | ||
|
|
bad73faf28 | ||
|
|
8d3704d93b | ||
|
|
8bd56fadd6 | ||
|
|
23f9b584a8 | ||
|
|
141557963e | ||
|
|
ffabf3c5b7 | ||
|
|
9b1e95809c | ||
|
|
b8776bb001 | ||
|
|
84c20e8337 | ||
|
|
bae2409cce | ||
|
|
8f7c44257b | ||
|
|
2772ce249c | ||
|
|
bcfd9327cb | ||
|
|
01ff36fc28 | ||
|
|
6466b09fdc | ||
|
|
40d7a701fe | ||
|
|
78b9d303d3 | ||
|
|
d912013d1b | ||
|
|
898e6519d1 | ||
|
|
8c074a7d01 | ||
|
|
6e69b4a795 | ||
|
|
1c36b46cb4 | ||
|
|
73abd04b95 | ||
|
|
29d6845d1f | ||
|
|
da84b26785 | ||
|
|
eb9070c833 | ||
|
|
b7b503d3ce | ||
|
|
327bbec981 | ||
|
|
7175a8e4eb | ||
|
|
9aefc183c6 | ||
|
|
3e9b867709 | ||
|
|
e8b5bbf7a1 | ||
|
|
79ef18e885 | ||
|
|
23a91df9f6 | ||
|
|
d8741595ae | ||
|
|
3c9dd6ffc2 | ||
|
|
42d28c9ff6 | ||
|
|
82372cbce8 | ||
|
|
69b07c5cc2 | ||
|
|
a6d27dc02c | ||
|
|
bbcaf48cb0 | ||
|
|
85194d99da | ||
|
|
5c86d2faa2 |
No files matched your search
@@ -8,52 +8,41 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: make
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get -y remove libzstd || true
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
|
||||
make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror'
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
|
||||
make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' BUILD_TLS=yes -j2
|
||||
- name: gen-cert
|
||||
run: ./utils/gen-test-certs.sh
|
||||
- name: test-tls
|
||||
run: |
|
||||
sudo apt-get -y install tcl tcl-tls
|
||||
./runtest --clients 1 --verbose --tls --config server-threads 3
|
||||
./runtest --clients 2 --verbose --tls
|
||||
- name: cluster-test
|
||||
run: |
|
||||
./runtest-cluster --tls --config server-threads 3
|
||||
./runtest-cluster --tls
|
||||
- name: sentinel test
|
||||
run: |
|
||||
./runtest-sentinel
|
||||
- name: module tests
|
||||
run: |
|
||||
./runtest-moduleapi
|
||||
- name: rotation test
|
||||
run: |
|
||||
./runtest-rotation
|
||||
|
||||
build-ubuntu-old:
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-16.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
- uses: actions/checkout@v1
|
||||
- name: make -j2
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get -y remove libzstd || true
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
|
||||
make -j2
|
||||
|
||||
build-macos-latest:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: make
|
||||
run: make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' -j2
|
||||
|
||||
@@ -61,11 +50,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: recursive
|
||||
- name: make
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get -y remove libzstd || true
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
|
||||
make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Endurance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '30 * * * *'
|
||||
|
||||
jobs:
|
||||
|
||||
test-endurance:
|
||||
runs-on: octocore
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v1
|
||||
- name: make
|
||||
run: |
|
||||
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
|
||||
make -j8
|
||||
- name: test-multithread (5X)
|
||||
run: |
|
||||
sudo apt-get install -y tcl tcl-tls
|
||||
./runtest --loopn 5 --config server-threads 3 --clients 5 --endurance
|
||||
|
||||
+2
-1
@@ -15,6 +15,7 @@ src/keydb-server
|
||||
**/app/keydb-server
|
||||
*.deb
|
||||
*.rpm
|
||||
keydb-pro-server
|
||||
src/keydb-cli
|
||||
**/bin/keydb-cli
|
||||
**/app/keydb-cli
|
||||
@@ -48,7 +49,7 @@ src/nodes.conf
|
||||
deps/lua/src/lua
|
||||
deps/lua/src/luac
|
||||
deps/lua/src/liblua.a
|
||||
tests/tls*/*
|
||||
tests/tls/*
|
||||
.make-*
|
||||
.prerequisites
|
||||
*.dSYM
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
[submodule "deps/rocksdb"]
|
||||
path = deps/rocksdb
|
||||
url = https://github.com/facebook/rocksdb.git
|
||||
[submodule "deps/depot_tools"]
|
||||
path = deps/depot_tools
|
||||
url = https://chromium.googlesource.com/chromium/tools/depot_tools.git
|
||||
-126
@@ -11,132 +11,6 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
|
||||
SECURITY: There are security fixes in the release.
|
||||
--------------------------------------------------------------------------------
|
||||
|
||||
================================================================================
|
||||
Redis 6.2.6 Released Mon Oct 4 12:00:00 IDT 2021
|
||||
================================================================================
|
||||
|
||||
Upgrade urgency: SECURITY, contains fixes to security issues.
|
||||
|
||||
Security Fixes:
|
||||
* (CVE-2021-41099) Integer to heap buffer overflow handling certain string
|
||||
commands and network payloads, when proto-max-bulk-len is manually configured
|
||||
to a non-default, very large value [reported by yiyuaner].
|
||||
* (CVE-2021-32762) Integer to heap buffer overflow issue in redis-cli and
|
||||
redis-sentinel parsing large multi-bulk replies on some older and less common
|
||||
platforms [reported by Microsoft Vulnerability Research].
|
||||
* (CVE-2021-32687) Integer to heap buffer overflow with intsets, when
|
||||
set-max-intset-entries is manually configured to a non-default, very large
|
||||
value [reported by Pawel Wieczorkiewicz, AWS].
|
||||
* (CVE-2021-32675) Denial Of Service when processing RESP request payloads with
|
||||
a large number of elements on many connections.
|
||||
* (CVE-2021-32672) Random heap reading issue with Lua Debugger [reported by
|
||||
Meir Shpilraien].
|
||||
* (CVE-2021-32628) Integer to heap buffer overflow handling ziplist-encoded
|
||||
data types, when configuring a large, non-default value for
|
||||
hash-max-ziplist-entries, hash-max-ziplist-value, zset-max-ziplist-entries
|
||||
or zset-max-ziplist-value [reported by sundb].
|
||||
* (CVE-2021-32627) Integer to heap buffer overflow issue with streams, when
|
||||
configuring a non-default, large value for proto-max-bulk-len and
|
||||
client-query-buffer-limit [reported by sundb].
|
||||
* (CVE-2021-32626) Specially crafted Lua scripts may result with Heap buffer
|
||||
overflow [reported by Meir Shpilraien].
|
||||
|
||||
Bug fixes that involve behavior changes:
|
||||
* GEO* STORE with empty source key deletes the destination key and return 0 (#9271)
|
||||
Previously it would have returned an empty array like the non-STORE variant.
|
||||
* PUBSUB NUMPAT replies with number of patterns rather than number of subscriptions (#9209)
|
||||
This actually changed in 6.2.0 but was overlooked and omitted from the release notes.
|
||||
|
||||
Bug fixes that are only applicable to previous releases of Redis 6.2:
|
||||
* Fix CLIENT PAUSE, used an old timeout from previous PAUSE (#9477)
|
||||
* Fix CLIENT PAUSE in a replica would mess the replication offset (#9448)
|
||||
* Add some missing error statistics in INFO errorstats (#9328)
|
||||
|
||||
Other bug fixes:
|
||||
* Fix incorrect reply of COMMAND command key positions for MIGRATE command (#9455)
|
||||
* Fix appendfsync to always guarantee fsync before reply, on MacOS and FreeBSD (kqueue) (#9416)
|
||||
* Fix the wrong mis-detection of sync_file_range system call, affecting performance (#9371)
|
||||
|
||||
CLI tools:
|
||||
* When redis-cli received ASK response, it didn't handle it (#8930)
|
||||
|
||||
Improvements:
|
||||
* Add latency monitor sample when key is deleted via lazy expire (#9317)
|
||||
* Sanitize corrupt payload improvements (#9321, #9399)
|
||||
* Delete empty keys when loading RDB file or handling a RESTORE command (#9297, #9349)
|
||||
|
||||
================================================================================
|
||||
Redis 6.2.5 Released Wed Jul 21 16:32:19 IDT 2021
|
||||
================================================================================
|
||||
|
||||
Upgrade urgency: SECURITY, contains fixes to security issues that affect
|
||||
authenticated client connections on 32-bit versions. MODERATE otherwise.
|
||||
|
||||
Fix integer overflow in BITFIELD on 32-bit versions (CVE-2021-32761).
|
||||
An integer overflow bug in Redis version 2.2 or newer can be exploited using the
|
||||
BITFIELD command to corrupt the heap and potentially result with remote code
|
||||
execution.
|
||||
|
||||
Bug fixes that involve behavior changes:
|
||||
* Change reply type for ZPOPMAX/MIN with count in RESP3 to nested array (#8981).
|
||||
Was using a flat array like in RESP2 instead of a nested array like ZRANGE does.
|
||||
* Fix reply type for HRANDFIELD and ZRANDMEMBER when key is missing (#9178).
|
||||
Was using a null array instead of an empty array.
|
||||
* Fix reply type for ZRANGESTORE when source key is missing (#9089).
|
||||
Was using an empty array like ZRANGE instead of 0 (used in the STORE variant).
|
||||
|
||||
Bug fixes that are only applicable to previous releases of Redis 6.2:
|
||||
* ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162)
|
||||
* Fix crash after CLIENT UNPAUSE when threaded I/O config is enabled (#9041)
|
||||
* Fix XTRIM or XADD with LIMIT may delete more entries than the limit (#9048)
|
||||
* Fix build issue with OpenSSL 1.1.0 (#9233)
|
||||
|
||||
Other bug fixes:
|
||||
* Fail EXEC command in case a watched key is expired (#9194)
|
||||
* Fix SMOVE not to invalidate dest key (WATCH and tracking) when member already exists (#9244)
|
||||
* Fix SINTERSTORE not to delete dest key when getting a wrong type error (#9032)
|
||||
* Fix overflows on 32-bit versions in GETBIT, SETBIT, BITCOUNT, BITPOS, and BITFIELD (#9191)
|
||||
* Improve MEMORY USAGE on stream keys (#9164)
|
||||
* Set TCP keepalive on inbound cluster bus connections (#9230)
|
||||
* Fix diskless replica loading to recover from RDB short read on module AUX data (#9199)
|
||||
* Fix race in client side tracking (#9116)
|
||||
* Fix ziplist length updates on big-endian platforms (#2080)
|
||||
|
||||
CLI tools:
|
||||
* redis-cli cluster import command may issue wrong MIGRATE command, sending COPY instead of REPLACE (#8945)
|
||||
* redis-cli --rdb fixes when using "-" to write to stdout (#9136, #9135)
|
||||
* redis-cli support for RESP3 set type in CSV and RAW output (#7338)
|
||||
|
||||
Modules:
|
||||
* Module API for getting current command name (#8792)
|
||||
* Fix RM_StringTruncate when newlen is 0 (#3718)
|
||||
* Fix CLIENT UNBLOCK crashing modules without timeout callback (#9167)
|
||||
|
||||
================================================================================
|
||||
Redis 6.2.4 Released Tue June 1 12:00:00 IST 2021
|
||||
================================================================================
|
||||
|
||||
Upgrade urgency: SECURITY, Contains fixes to security issues that affect
|
||||
authenticated client connections. MODERATE otherwise.
|
||||
|
||||
Fix integer overflow in STRALGO LCS (CVE-2021-32625)
|
||||
An integer overflow bug in Redis version 6.0 or newer can be exploited using the
|
||||
STRALGO LCS command to corrupt the heap and potentially result with remote code
|
||||
execution. This is a result of an incomplete fix by CVE-2021-29477.
|
||||
|
||||
Bug fixes that are only applicable to previous releases of Redis 6.2:
|
||||
* Fix crash after a diskless replication fork child is terminated (#8991)
|
||||
* Fix redis-benchmark crash on unsupported configs (#8916)
|
||||
|
||||
Other bug fixes:
|
||||
* Fix crash in UNLINK on a stream key with deleted consumer groups (#8932)
|
||||
* SINTERSTORE: Add missing keyspace del event when none of the sources exist (#8949)
|
||||
* Sentinel: Fix CONFIG SET of empty string sentinel-user/sentinel-pass configs (#8958)
|
||||
* Enforce client output buffer soft limit when no traffic (#8833)
|
||||
|
||||
Improvements:
|
||||
* Hide AUTH passwords in MIGRATE command from slowlog (#8859)
|
||||
|
||||
================================================================================
|
||||
Redis 6.2.3 Released Mon May 3 19:00:00 IST 2021
|
||||
================================================================================
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
Copyright (c) 2006-2020, Salvatore Sanfilippo
|
||||
Copyright (C) 2019-2021, John Sully
|
||||
Copyright (C) 2020-2021, EQ Alpha Technology Ltd.
|
||||
Copyright (C) 2022 Snap Inc.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Top level makefile, this just calls into src/Makefile where the real work is done. Changes should be made there.
|
||||
# Top level makefile, the real shit is at src/Makefile
|
||||
|
||||
default: all
|
||||
|
||||
|
||||
@@ -2,20 +2,16 @@
|
||||

|
||||
[](https://stackshare.io/eq-alpha-technology-inc/eq-alpha-technology-inc)
|
||||
|
||||
##### KeyDB is now a part of Snap Inc! Check out the announcement [here](https://docs.keydb.dev/news/2022/05/12/keydb-joins-snap)
|
||||
|
||||
##### [Release v6.3.0](https://github.com/EQ-Alpha/KeyDB/releases/tag/v6.3.0) is here with major improvements as we consolidate our Open Source and Enterprise offerings into a single BSD-3 licensed project. See our [roadmap](https://docs.keydb.dev/docs/coming-soon) for details.
|
||||
|
||||
##### Want to extend KeyDB with Javascript? Try [ModJS](https://github.com/JohnSully/ModJS)
|
||||
|
||||
##### Need Help? Check out our extensive [documentation](https://docs.keydb.dev).
|
||||
|
||||
##### KeyDB is on Slack. Click [here](https://docs.keydb.dev/slack/) to learn more and join the KeyDB Community Slack workspace.
|
||||
##### NEW!!! KeyDB now has a Slack Community Workspace. Click [here](https://docs.keydb.dev/slack/) to learn more and join the KeyDB Community Slack workspace.
|
||||
|
||||
What is KeyDB?
|
||||
--------------
|
||||
|
||||
KeyDB is a high performance fork of Redis with a focus on multithreading, memory efficiency, and high throughput. In addition to performance improvements, KeyDB offers features such as Active Replication, FLASH Storage and Subkey Expires. KeyDB has a MVCC architecture that allows you to execute queries such as KEYS and SCAN without blocking the database and degrading performance.
|
||||
KeyDB is a high performance fork of Redis with a focus on multithreading, memory efficiency, and high throughput. In addition to multithreading, KeyDB also has features only available in Redis Enterprise such as [Active Replication](https://github.com/JohnSully/KeyDB/wiki/Active-Replication), [FLASH storage](https://github.com/JohnSully/KeyDB/wiki/FLASH-Storage) support, and some not available at all such as direct backup to AWS S3.
|
||||
|
||||
KeyDB maintains full compatibility with the Redis protocol, modules, and scripts. This includes the atomicity guarantees for scripts and transactions. Because KeyDB keeps in sync with Redis development KeyDB is a superset of Redis functionality, making KeyDB a drop in replacement for existing Redis deployments.
|
||||
|
||||
@@ -34,81 +30,54 @@ KeyDB has a different philosophy on how the codebase should evolve. We feel tha
|
||||
|
||||
Because of this difference of opinion features which are right for KeyDB may not be appropriate for Redis. A fork allows us to explore this new development path and implement features which may never be a part of Redis. KeyDB keeps in sync with upstream Redis changes, and where applicable we upstream bug fixes and changes. It is our hope that the two projects can continue to grow and learn from each other.
|
||||
|
||||
Project Support
|
||||
-------------------
|
||||
|
||||
The KeyDB team maintains this project as part of Snap Inc. KeyDB is used by Snap as part of its caching infrastructure and is fully open sourced. There is no separate commercial product and no paid support options available. We really value collaborating with the open source community and welcome PRs, bug reports, and open discussion. For community support or to get involved further with the project check out our community support options [here](https://docs.keydb.dev/docs/support) (slack, forum, meetup, github issues). Our team monitors these channels regularly.
|
||||
|
||||
|
||||
Additional Resources
|
||||
--------------------
|
||||
|
||||
Try the KeyDB [Docker Image](https://hub.docker.com/r/eqalpha/keydb)
|
||||
Try our docker container: https://hub.docker.com/r/eqalpha/keydb
|
||||
|
||||
Join us on [Slack](https://docs.keydb.dev/slack/)
|
||||
Talk on Gitter: https://gitter.im/KeyDB
|
||||
|
||||
Learn more using KeyDB's extensive [documentation](https://docs.keydb.dev)
|
||||
Visit our Website: https://keydb.dev
|
||||
|
||||
Post to our [Community Forum](https://community.keydb.dev)
|
||||
See options for channel partners and support contracts: https://keydb.dev/support.html
|
||||
|
||||
See the [KeyDB Roadmap](https://docs.keydb.dev/docs/coming-soon) to see what's in store
|
||||
Learn with KeyDB’s official documentation site: https://docs.keydb.dev
|
||||
|
||||
[Subscribe to the KeyDB mailing list](https://eqalpha.us20.list-manage.com/subscribe/post?u=978f486c2f95589b24591a9cc&id=4ab9220500)
|
||||
|
||||
Management GUI: We recommend [FastoNoSQL](https://fastonosql.com/) which has official KeyDB support.
|
||||
|
||||
|
||||
Benchmarking KeyDB
|
||||
------------------
|
||||
|
||||
Please note keydb-benchmark and redis-benchmark are currently single threaded and too slow to properly benchmark KeyDB. We recommend using a redis cluster benchmark tool such as [memtier](https://github.com/RedisLabs/memtier_benchmark). Please ensure your machine has enough cores for both KeyDB and memtier if testing locally. KeyDB expects exclusive use of any cores assigned to it.
|
||||
Please note keydb-benchmark and redis-benchmark are currently single threaded and too slow to properly benchmark KeyDB. We recommend using a redis cluster benchmark tool such as [memtier](https://github.com/RedisLabs/memtier_benchmark). Please ensure your machine has enough cores for both KeyDB and memteir if testing locally. KeyDB expects exclusive use of any cores assigned to it.
|
||||
|
||||
For more details on how we benchmarked KeyDB along with performance numbers check out our blog post: [Redis Should Be Multithreaded](https://medium.com/@john_63123/redis-should-be-multi-threaded-e28319cab744?source=friends_link&sk=7ce8e9fe3ec8224a4d27ef075d085457)
|
||||
|
||||
New Configuration Options
|
||||
-------------------------
|
||||
|
||||
With new features comes new options. All other configuration options behave as you'd expect. Your existing configuration files should continue to work unchanged.
|
||||
With new features comes new options:
|
||||
|
||||
```
|
||||
server-threads N
|
||||
server-thread-affinity [true/false]
|
||||
```
|
||||
The number of threads used to serve requests. This should be related to the number of queues available in your network hardware, *not* the number of cores on your
|
||||
machine. Because KeyDB uses spinlocks to reduce latency; making this too high will reduce performance. We recommend using 4 here. By default this is set to two.
|
||||
|
||||
```
|
||||
min-clients-per-thread 50
|
||||
```
|
||||
The minimum number of clients on a thread before KeyDB assigns new connections to a different thread. Tuning this parameter is a tradeoff between locking overhead and distributing the workload over multiple cores
|
||||
The number of threads used to serve requests. This should be related to the number of queues available in your network hardware, *not* the number of cores on your machine. Because KeyDB uses spinlocks to reduce latency; making this too high will reduce performance. We recommend using 4 here. By default this is set to one.
|
||||
|
||||
```
|
||||
replica-weighting-factor 2
|
||||
```
|
||||
KeyDB will attempt to balance clients across threads evenly; However, replica clients are usually much more expensive than a normal client, and so KeyDB will try to assign fewer clients to threads with a replica. The weighting factor below is intented to help tune this behavior. A replica weighting factor of 2 means we treat a replica as the equivalent of two normal clients. Adjusting this value may improve performance when replication is used. The best weighting is workload specific - e.g. read heavy workloads should set this to 1. Very write heavy workloads may benefit from higher numbers.
|
||||
scratch-file-path /path
|
||||
|
||||
```
|
||||
active-client-balancing yes
|
||||
```
|
||||
Should KeyDB make active attempts at balancing clients across threads? This can impact performance accepting new clients. By default this is enabled. If disabled there is still a best effort from the kernel to distribute across threads with SO_REUSEPORT but it will not be as fair. By default this is enabled
|
||||
|
||||
```
|
||||
active-replica yes
|
||||
```
|
||||
If you are using active-active replication set `active-replica` option to “yes”. This will enable both instances to accept reads and writes while remaining synced. [Click here](https://docs.keydb.dev/docs/active-rep/) to see more on active-rep in our docs section. There are also [docker examples]( https://docs.keydb.dev/docs/docker-active-rep/) on docs.
|
||||
|
||||
```
|
||||
multi-master-no-forward no
|
||||
```
|
||||
Avoid forwarding RREPLAY messages to other masters? WARNING: This setting is dangerous! You must be certain all masters are connected to eachother in a true mesh topology or data loss will occur! This command can be used to reduce multimaster bus traffic
|
||||
|
||||
|
||||
```
|
||||
If you would like to use the [FLASH backed](https://github.com/JohnSully/KeyDB/wiki/FLASH-Storage) storage this option configures the directory for KeyDB's temporary files. This feature relies on snapshotting to work so must be used on a BTRFS filesystem. ZFS may also work but is untested. With this feature KeyDB will use RAM as a cache and page to disk as necessary. NOTE: This requires special compilation options, see Building KeyDB below.
|
||||
|
||||
db-s3-object /path/to/bucket
|
||||
```
|
||||
|
||||
If you would like KeyDB to dump and load directly to AWS S3 this option specifies the bucket. Using this option with the traditional RDB options will result in KeyDB backing up twice to both locations. If both are specified KeyDB will first attempt to load from the local dump file and if that fails load from S3. This requires the AWS CLI tools to be installed and configured which are used under the hood to transfer the data.
|
||||
|
||||
active-replica yes
|
||||
|
||||
```
|
||||
storage-provider flash /path/to/flash
|
||||
```
|
||||
If you would like to use KeyDB FLASH storage, specify the storage medium followed by the directory path on your local SSD volume. Note that this feature is still considered experimental and should be used with discretion. See [FLASH Documentation](https://docs.keydb.dev/docs/flash) for more details on configuration and setting up your FLASH volume.
|
||||
If you are using active-active replication set `active-replica` option to “yes”. This will enable both instances to accept reads and writes while remaining synced. [Click here](https://docs.keydb.dev/docs/active-rep/) to see more on active-rep in our docs section. There are also [docker examples]( https://docs.keydb.dev/docs/docker-active-rep/) on docs.
|
||||
|
||||
All other configuration options behave as you'd expect. Your existing configuration files should continue to work unchanged.
|
||||
|
||||
Building KeyDB
|
||||
--------------
|
||||
@@ -117,18 +86,19 @@ KeyDB can be compiled and is tested for use on Linux. KeyDB currently relies on
|
||||
|
||||
More on CentOS/Archlinux/Alpine/Debian/Ubuntu dependencies and builds can be found here: https://docs.keydb.dev/docs/build/
|
||||
|
||||
Init and clone submodule dependencies:
|
||||
|
||||
% git submodule init && git submodule update
|
||||
|
||||
Install dependencies:
|
||||
|
||||
% sudo apt install build-essential nasm autotools-dev autoconf libjemalloc-dev tcl tcl-dev uuid-dev libcurl4-openssl-dev libbz2-dev libzstd-dev liblz4-dev libsnappy-dev libssl-dev
|
||||
% sudo apt install build-essential nasm autotools-dev autoconf libjemalloc-dev tcl tcl-dev uuid-dev libcurl4-openssl-dev
|
||||
|
||||
Compiling is as simple as:
|
||||
|
||||
% make
|
||||
|
||||
To build with TLS support, you'll need OpenSSL development libraries (e.g.
|
||||
libssl-dev on Debian/Ubuntu) and run:
|
||||
|
||||
% make BUILD_TLS=yes
|
||||
|
||||
To build with systemd support, you'll need systemd development libraries (such
|
||||
as libsystemd-dev on Debian/Ubuntu or systemd-devel on CentOS) and run:
|
||||
|
||||
@@ -141,21 +111,18 @@ To append a suffix to KeyDB program names, use:
|
||||
***Note that the following dependencies may be needed:
|
||||
% sudo apt-get install autoconf autotools-dev libnuma-dev libtool
|
||||
|
||||
To buik=ld with TLS support, use:
|
||||
|
||||
% make BUILD_TLS=yes
|
||||
|
||||
Running the tests with TLS enabled (you will need `tcl-tls`
|
||||
If TLS is built, running the tests with TLS enabled (you will need `tcl-tls`
|
||||
installed):
|
||||
|
||||
% ./utils/gen-test-certs.sh
|
||||
% ./runtest --tls
|
||||
|
||||
To build with KeyDB FLASH support, use:
|
||||
|
||||
% make ENABLE_FLASH=yes
|
||||
If TLS is built, running the tests with TLS enabled (you will need `tcl-tls`
|
||||
installed):
|
||||
|
||||
***Note that the KeyDB FLASH feature is considered experimental (beta) and should used with discretion
|
||||
% ./utils/gen-test-certs.sh
|
||||
% ./runtest --tls
|
||||
|
||||
|
||||
Fixing build problems with dependencies or cached build options
|
||||
@@ -253,11 +220,11 @@ All the options in keydb.conf are also supported as options using the command
|
||||
line, with exactly the same name.
|
||||
|
||||
|
||||
Running KeyDB with TLS:
|
||||
Running Redis with TLS:
|
||||
------------------
|
||||
|
||||
Please consult the [TLS.md](TLS.md) file for more information on
|
||||
how to use KeyDB with TLS.
|
||||
how to use Redis with TLS.
|
||||
|
||||
|
||||
Playing with KeyDB
|
||||
@@ -317,6 +284,24 @@ KeyDB works by running the normal Redis event loop on multiple threads. Network
|
||||
|
||||
Unlike most databases the core data structure is the fastest part of the system. Most of the query time comes from parsing the REPL protocol and copying data to/from the network.
|
||||
|
||||
Future work:
|
||||
- Allow rebalancing of connections to different threads after the connection
|
||||
- Allow multiple readers access to the hashtable concurrently
|
||||
|
||||
Docker Build
|
||||
------------
|
||||
Build the latest binaries from the github unstable branch within a docker container. Note this is built for Ubuntu 18.04.
|
||||
Simply make a directory you would like to have the latest binaries dumped in, then run the following commmand with your updated path:
|
||||
```
|
||||
$ docker run -it --rm -v /path-to-dump-binaries:/keydb_bin eqalpha/keydb-build-bin
|
||||
```
|
||||
You should receive the following files: keydb-benchmark, keydb-check-aof, keydb-check-rdb, keydb-cli, keydb-sentinel, keydb-server
|
||||
|
||||
If you are looking to enable flash support with the build (make MALLOC=memkind) then use the following command:
|
||||
```
|
||||
$ docker run -it --rm -v /path-to-dump-binaries:/keydb_bin eqalpha/keydb-build-bin:flash
|
||||
```
|
||||
Please note that you will need libcurl4-openssl-dev in order to run keydb. With flash version you may need libnuma-dev and libtool installed in order to run the binaries. Keep this in mind especially when running in a container. For a copy of all our Dockerfiles, please see them on [docs]( https://docs.keydb.dev/docs/dockerfiles/).
|
||||
|
||||
Code contributions
|
||||
-----------------
|
||||
|
||||
@@ -6,7 +6,10 @@ Getting Started
|
||||
|
||||
### Building
|
||||
|
||||
TLS support is enabled in the default build. To build without TLS, run `make BUILD_TLS=no`.
|
||||
To build with TLS support you'll need OpenSSL development libraries (e.g.
|
||||
libssl-dev on Debian/Ubuntu).
|
||||
|
||||
Run `make BUILD_TLS=yes`.
|
||||
|
||||
### Tests
|
||||
|
||||
|
||||
Vendored
+1
-13
@@ -1,7 +1,6 @@
|
||||
# Redis dependency Makefile
|
||||
|
||||
uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not')
|
||||
uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
|
||||
|
||||
CCCOLOR="\033[34m"
|
||||
LINKCOLOR="\033[34;1m"
|
||||
@@ -38,7 +37,6 @@ distclean:
|
||||
-(cd linenoise && $(MAKE) clean) > /dev/null || true
|
||||
-(cd lua && $(MAKE) clean) > /dev/null || true
|
||||
-(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true
|
||||
-(cd rocksdb && $(MAKE) clean) > /dev/null || true
|
||||
-(cd hdr_histogram && $(MAKE) clean) > /dev/null || true
|
||||
-(rm -f .make-*)
|
||||
|
||||
@@ -95,17 +93,7 @@ JEMALLOC_LDFLAGS= $(LDFLAGS)
|
||||
|
||||
jemalloc: .make-prerequisites
|
||||
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
||||
cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --disable-cxx CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
|
||||
cd jemalloc && ./configure --with-version=5.2.1-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
|
||||
cd jemalloc && $(MAKE) CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" lib/libjemalloc.a
|
||||
|
||||
.PHONY: jemalloc
|
||||
|
||||
rocksdb: .make-prerequisites
|
||||
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
|
||||
ifeq ($(uname_M),x86_64)
|
||||
cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib
|
||||
else
|
||||
cd rocksdb && PORTABLE=1 $(MAKE) static_lib
|
||||
endif
|
||||
|
||||
.PHONY: rocksdb
|
||||
-583
@@ -1,583 +0,0 @@
|
||||
// Provides an efficient blocking version of moodycamel::ConcurrentQueue.
|
||||
// ©2015-2020 Cameron Desrochers. Distributed under the terms of the simplified
|
||||
// BSD license, available at the top of concurrentqueue.h.
|
||||
// Also dual-licensed under the Boost Software License (see LICENSE.md)
|
||||
// Uses Jeff Preshing's semaphore implementation (under the terms of its
|
||||
// separate zlib license, see lightweightsemaphore.h).
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "concurrentqueue.h"
|
||||
#include "lightweightsemaphore.h"
|
||||
|
||||
#include <type_traits>
|
||||
#include <cerrno>
|
||||
#include <memory>
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
|
||||
namespace moodycamel
|
||||
{
|
||||
// This is a blocking version of the queue. It has an almost identical interface to
|
||||
// the normal non-blocking version, with the addition of various wait_dequeue() methods
|
||||
// and the removal of producer-specific dequeue methods.
|
||||
template<typename T, typename Traits = ConcurrentQueueDefaultTraits>
|
||||
class BlockingConcurrentQueue
|
||||
{
|
||||
private:
|
||||
typedef ::moodycamel::ConcurrentQueue<T, Traits> ConcurrentQueue;
|
||||
typedef ::moodycamel::LightweightSemaphore LightweightSemaphore;
|
||||
|
||||
public:
|
||||
typedef typename ConcurrentQueue::producer_token_t producer_token_t;
|
||||
typedef typename ConcurrentQueue::consumer_token_t consumer_token_t;
|
||||
|
||||
typedef typename ConcurrentQueue::index_t index_t;
|
||||
typedef typename ConcurrentQueue::size_t size_t;
|
||||
typedef typename std::make_signed<size_t>::type ssize_t;
|
||||
|
||||
static const size_t BLOCK_SIZE = ConcurrentQueue::BLOCK_SIZE;
|
||||
static const size_t EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD = ConcurrentQueue::EXPLICIT_BLOCK_EMPTY_COUNTER_THRESHOLD;
|
||||
static const size_t EXPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::EXPLICIT_INITIAL_INDEX_SIZE;
|
||||
static const size_t IMPLICIT_INITIAL_INDEX_SIZE = ConcurrentQueue::IMPLICIT_INITIAL_INDEX_SIZE;
|
||||
static const size_t INITIAL_IMPLICIT_PRODUCER_HASH_SIZE = ConcurrentQueue::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE;
|
||||
static const std::uint32_t EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE = ConcurrentQueue::EXPLICIT_CONSUMER_CONSUMPTION_QUOTA_BEFORE_ROTATE;
|
||||
static const size_t MAX_SUBQUEUE_SIZE = ConcurrentQueue::MAX_SUBQUEUE_SIZE;
|
||||
|
||||
public:
|
||||
// Creates a queue with at least `capacity` element slots; note that the
|
||||
// actual number of elements that can be inserted without additional memory
|
||||
// allocation depends on the number of producers and the block size (e.g. if
|
||||
// the block size is equal to `capacity`, only a single block will be allocated
|
||||
// up-front, which means only a single producer will be able to enqueue elements
|
||||
// without an extra allocation -- blocks aren't shared between producers).
|
||||
// This method is not thread safe -- it is up to the user to ensure that the
|
||||
// queue is fully constructed before it starts being used by other threads (this
|
||||
// includes making the memory effects of construction visible, possibly with a
|
||||
// memory barrier).
|
||||
explicit BlockingConcurrentQueue(size_t capacity = 6 * BLOCK_SIZE)
|
||||
: inner(capacity), sema(create<LightweightSemaphore, ssize_t, int>(0, (int)Traits::MAX_SEMA_SPINS), &BlockingConcurrentQueue::template destroy<LightweightSemaphore>)
|
||||
{
|
||||
assert(reinterpret_cast<ConcurrentQueue*>((BlockingConcurrentQueue*)1) == &((BlockingConcurrentQueue*)1)->inner && "BlockingConcurrentQueue must have ConcurrentQueue as its first member");
|
||||
if (!sema) {
|
||||
MOODYCAMEL_THROW(std::bad_alloc());
|
||||
}
|
||||
}
|
||||
|
||||
BlockingConcurrentQueue(size_t minCapacity, size_t maxExplicitProducers, size_t maxImplicitProducers)
|
||||
: inner(minCapacity, maxExplicitProducers, maxImplicitProducers), sema(create<LightweightSemaphore, ssize_t, int>(0, (int)Traits::MAX_SEMA_SPINS), &BlockingConcurrentQueue::template destroy<LightweightSemaphore>)
|
||||
{
|
||||
assert(reinterpret_cast<ConcurrentQueue*>((BlockingConcurrentQueue*)1) == &((BlockingConcurrentQueue*)1)->inner && "BlockingConcurrentQueue must have ConcurrentQueue as its first member");
|
||||
if (!sema) {
|
||||
MOODYCAMEL_THROW(std::bad_alloc());
|
||||
}
|
||||
}
|
||||
|
||||
// Disable copying and copy assignment
|
||||
BlockingConcurrentQueue(BlockingConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION;
|
||||
BlockingConcurrentQueue& operator=(BlockingConcurrentQueue const&) MOODYCAMEL_DELETE_FUNCTION;
|
||||
|
||||
// Moving is supported, but note that it is *not* a thread-safe operation.
|
||||
// Nobody can use the queue while it's being moved, and the memory effects
|
||||
// of that move must be propagated to other threads before they can use it.
|
||||
// Note: When a queue is moved, its tokens are still valid but can only be
|
||||
// used with the destination queue (i.e. semantically they are moved along
|
||||
// with the queue itself).
|
||||
BlockingConcurrentQueue(BlockingConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT
|
||||
: inner(std::move(other.inner)), sema(std::move(other.sema))
|
||||
{ }
|
||||
|
||||
inline BlockingConcurrentQueue& operator=(BlockingConcurrentQueue&& other) MOODYCAMEL_NOEXCEPT
|
||||
{
|
||||
return swap_internal(other);
|
||||
}
|
||||
|
||||
// Swaps this queue's state with the other's. Not thread-safe.
|
||||
// Swapping two queues does not invalidate their tokens, however
|
||||
// the tokens that were created for one queue must be used with
|
||||
// only the swapped queue (i.e. the tokens are tied to the
|
||||
// queue's movable state, not the object itself).
|
||||
inline void swap(BlockingConcurrentQueue& other) MOODYCAMEL_NOEXCEPT
|
||||
{
|
||||
swap_internal(other);
|
||||
}
|
||||
|
||||
private:
|
||||
BlockingConcurrentQueue& swap_internal(BlockingConcurrentQueue& other)
|
||||
{
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
|
||||
inner.swap(other.inner);
|
||||
sema.swap(other.sema);
|
||||
return *this;
|
||||
}
|
||||
|
||||
public:
|
||||
// Enqueues a single item (by copying it).
|
||||
// Allocates memory if required. Only fails if memory allocation fails (or implicit
|
||||
// production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0,
|
||||
// or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Thread-safe.
|
||||
inline bool enqueue(T const& item)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue(item))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by moving it, if possible).
|
||||
// Allocates memory if required. Only fails if memory allocation fails (or implicit
|
||||
// production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0,
|
||||
// or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Thread-safe.
|
||||
inline bool enqueue(T&& item)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue(std::move(item)))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by copying it) using an explicit producer token.
|
||||
// Allocates memory if required. Only fails if memory allocation fails (or
|
||||
// Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Thread-safe.
|
||||
inline bool enqueue(producer_token_t const& token, T const& item)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue(token, item))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by moving it, if possible) using an explicit producer token.
|
||||
// Allocates memory if required. Only fails if memory allocation fails (or
|
||||
// Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Thread-safe.
|
||||
inline bool enqueue(producer_token_t const& token, T&& item)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue(token, std::move(item)))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues several items.
|
||||
// Allocates memory if required. Only fails if memory allocation fails (or
|
||||
// implicit production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
|
||||
// is 0, or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Note: Use std::make_move_iterator if the elements should be moved instead of copied.
|
||||
// Thread-safe.
|
||||
template<typename It>
|
||||
inline bool enqueue_bulk(It itemFirst, size_t count)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue_bulk(std::forward<It>(itemFirst), count))) {
|
||||
sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues several items using an explicit producer token.
|
||||
// Allocates memory if required. Only fails if memory allocation fails
|
||||
// (or Traits::MAX_SUBQUEUE_SIZE has been defined and would be surpassed).
|
||||
// Note: Use std::make_move_iterator if the elements should be moved
|
||||
// instead of copied.
|
||||
// Thread-safe.
|
||||
template<typename It>
|
||||
inline bool enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count)
|
||||
{
|
||||
if ((details::likely)(inner.enqueue_bulk(token, std::forward<It>(itemFirst), count))) {
|
||||
sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by copying it).
|
||||
// Does not allocate memory. Fails if not enough room to enqueue (or implicit
|
||||
// production is disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE
|
||||
// is 0).
|
||||
// Thread-safe.
|
||||
inline bool try_enqueue(T const& item)
|
||||
{
|
||||
if (inner.try_enqueue(item)) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by moving it, if possible).
|
||||
// Does not allocate memory (except for one-time implicit producer).
|
||||
// Fails if not enough room to enqueue (or implicit production is
|
||||
// disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0).
|
||||
// Thread-safe.
|
||||
inline bool try_enqueue(T&& item)
|
||||
{
|
||||
if (inner.try_enqueue(std::move(item))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by copying it) using an explicit producer token.
|
||||
// Does not allocate memory. Fails if not enough room to enqueue.
|
||||
// Thread-safe.
|
||||
inline bool try_enqueue(producer_token_t const& token, T const& item)
|
||||
{
|
||||
if (inner.try_enqueue(token, item)) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues a single item (by moving it, if possible) using an explicit producer token.
|
||||
// Does not allocate memory. Fails if not enough room to enqueue.
|
||||
// Thread-safe.
|
||||
inline bool try_enqueue(producer_token_t const& token, T&& item)
|
||||
{
|
||||
if (inner.try_enqueue(token, std::move(item))) {
|
||||
sema->signal();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues several items.
|
||||
// Does not allocate memory (except for one-time implicit producer).
|
||||
// Fails if not enough room to enqueue (or implicit production is
|
||||
// disabled because Traits::INITIAL_IMPLICIT_PRODUCER_HASH_SIZE is 0).
|
||||
// Note: Use std::make_move_iterator if the elements should be moved
|
||||
// instead of copied.
|
||||
// Thread-safe.
|
||||
template<typename It>
|
||||
inline bool try_enqueue_bulk(It itemFirst, size_t count)
|
||||
{
|
||||
if (inner.try_enqueue_bulk(std::forward<It>(itemFirst), count)) {
|
||||
sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enqueues several items using an explicit producer token.
|
||||
// Does not allocate memory. Fails if not enough room to enqueue.
|
||||
// Note: Use std::make_move_iterator if the elements should be moved
|
||||
// instead of copied.
|
||||
// Thread-safe.
|
||||
template<typename It>
|
||||
inline bool try_enqueue_bulk(producer_token_t const& token, It itemFirst, size_t count)
|
||||
{
|
||||
if (inner.try_enqueue_bulk(token, std::forward<It>(itemFirst), count)) {
|
||||
sema->signal((LightweightSemaphore::ssize_t)(ssize_t)count);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// Attempts to dequeue from the queue.
|
||||
// Returns false if all producer streams appeared empty at the time they
|
||||
// were checked (so, the queue is likely but not guaranteed to be empty).
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline bool try_dequeue(U& item)
|
||||
{
|
||||
if (sema->tryWait()) {
|
||||
while (!inner.try_dequeue(item)) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Attempts to dequeue from the queue using an explicit consumer token.
|
||||
// Returns false if all producer streams appeared empty at the time they
|
||||
// were checked (so, the queue is likely but not guaranteed to be empty).
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline bool try_dequeue(consumer_token_t& token, U& item)
|
||||
{
|
||||
if (sema->tryWait()) {
|
||||
while (!inner.try_dequeue(token, item)) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue.
|
||||
// Returns the number of items actually dequeued.
|
||||
// Returns 0 if all producer streams appeared empty at the time they
|
||||
// were checked (so, the queue is likely but not guaranteed to be empty).
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t try_dequeue_bulk(It itemFirst, size_t max)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->tryWaitMany((LightweightSemaphore::ssize_t)(ssize_t)max);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue using an explicit consumer token.
|
||||
// Returns the number of items actually dequeued.
|
||||
// Returns 0 if all producer streams appeared empty at the time they
|
||||
// were checked (so, the queue is likely but not guaranteed to be empty).
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t try_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->tryWaitMany((LightweightSemaphore::ssize_t)(ssize_t)max);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(token, itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Blocks the current thread until there's something to dequeue, then
|
||||
// dequeues it.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline void wait_dequeue(U& item)
|
||||
{
|
||||
while (!sema->wait()) {
|
||||
continue;
|
||||
}
|
||||
while (!inner.try_dequeue(item)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Blocks the current thread until either there's something to dequeue
|
||||
// or the timeout (specified in microseconds) expires. Returns false
|
||||
// without setting `item` if the timeout expires, otherwise assigns
|
||||
// to `item` and returns true.
|
||||
// Using a negative timeout indicates an indefinite timeout,
|
||||
// and is thus functionally equivalent to calling wait_dequeue.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline bool wait_dequeue_timed(U& item, std::int64_t timeout_usecs)
|
||||
{
|
||||
if (!sema->wait(timeout_usecs)) {
|
||||
return false;
|
||||
}
|
||||
while (!inner.try_dequeue(item)) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Blocks the current thread until either there's something to dequeue
|
||||
// or the timeout expires. Returns false without setting `item` if the
|
||||
// timeout expires, otherwise assigns to `item` and returns true.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U, typename Rep, typename Period>
|
||||
inline bool wait_dequeue_timed(U& item, std::chrono::duration<Rep, Period> const& timeout)
|
||||
{
|
||||
return wait_dequeue_timed(item, std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
|
||||
}
|
||||
|
||||
// Blocks the current thread until there's something to dequeue, then
|
||||
// dequeues it using an explicit consumer token.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline void wait_dequeue(consumer_token_t& token, U& item)
|
||||
{
|
||||
while (!sema->wait()) {
|
||||
continue;
|
||||
}
|
||||
while (!inner.try_dequeue(token, item)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Blocks the current thread until either there's something to dequeue
|
||||
// or the timeout (specified in microseconds) expires. Returns false
|
||||
// without setting `item` if the timeout expires, otherwise assigns
|
||||
// to `item` and returns true.
|
||||
// Using a negative timeout indicates an indefinite timeout,
|
||||
// and is thus functionally equivalent to calling wait_dequeue.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U>
|
||||
inline bool wait_dequeue_timed(consumer_token_t& token, U& item, std::int64_t timeout_usecs)
|
||||
{
|
||||
if (!sema->wait(timeout_usecs)) {
|
||||
return false;
|
||||
}
|
||||
while (!inner.try_dequeue(token, item)) {
|
||||
continue;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Blocks the current thread until either there's something to dequeue
|
||||
// or the timeout expires. Returns false without setting `item` if the
|
||||
// timeout expires, otherwise assigns to `item` and returns true.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename U, typename Rep, typename Period>
|
||||
inline bool wait_dequeue_timed(consumer_token_t& token, U& item, std::chrono::duration<Rep, Period> const& timeout)
|
||||
{
|
||||
return wait_dequeue_timed(token, item, std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue.
|
||||
// Returns the number of items actually dequeued, which will
|
||||
// always be at least one (this method blocks until the queue
|
||||
// is non-empty) and at most max.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t wait_dequeue_bulk(It itemFirst, size_t max)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue.
|
||||
// Returns the number of items actually dequeued, which can
|
||||
// be 0 if the timeout expires while waiting for elements,
|
||||
// and at most max.
|
||||
// Using a negative timeout indicates an indefinite timeout,
|
||||
// and is thus functionally equivalent to calling wait_dequeue_bulk.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t wait_dequeue_bulk_timed(It itemFirst, size_t max, std::int64_t timeout_usecs)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max, timeout_usecs);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue.
|
||||
// Returns the number of items actually dequeued, which can
|
||||
// be 0 if the timeout expires while waiting for elements,
|
||||
// and at most max.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It, typename Rep, typename Period>
|
||||
inline size_t wait_dequeue_bulk_timed(It itemFirst, size_t max, std::chrono::duration<Rep, Period> const& timeout)
|
||||
{
|
||||
return wait_dequeue_bulk_timed<It&>(itemFirst, max, std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue using an explicit consumer token.
|
||||
// Returns the number of items actually dequeued, which will
|
||||
// always be at least one (this method blocks until the queue
|
||||
// is non-empty) and at most max.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t wait_dequeue_bulk(consumer_token_t& token, It itemFirst, size_t max)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(token, itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue using an explicit consumer token.
|
||||
// Returns the number of items actually dequeued, which can
|
||||
// be 0 if the timeout expires while waiting for elements,
|
||||
// and at most max.
|
||||
// Using a negative timeout indicates an indefinite timeout,
|
||||
// and is thus functionally equivalent to calling wait_dequeue_bulk.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It>
|
||||
inline size_t wait_dequeue_bulk_timed(consumer_token_t& token, It itemFirst, size_t max, std::int64_t timeout_usecs)
|
||||
{
|
||||
size_t count = 0;
|
||||
max = (size_t)sema->waitMany((LightweightSemaphore::ssize_t)(ssize_t)max, timeout_usecs);
|
||||
while (count != max) {
|
||||
count += inner.template try_dequeue_bulk<It&>(token, itemFirst, max - count);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Attempts to dequeue several elements from the queue using an explicit consumer token.
|
||||
// Returns the number of items actually dequeued, which can
|
||||
// be 0 if the timeout expires while waiting for elements,
|
||||
// and at most max.
|
||||
// Never allocates. Thread-safe.
|
||||
template<typename It, typename Rep, typename Period>
|
||||
inline size_t wait_dequeue_bulk_timed(consumer_token_t& token, It itemFirst, size_t max, std::chrono::duration<Rep, Period> const& timeout)
|
||||
{
|
||||
return wait_dequeue_bulk_timed<It&>(token, itemFirst, max, std::chrono::duration_cast<std::chrono::microseconds>(timeout).count());
|
||||
}
|
||||
|
||||
|
||||
// Returns an estimate of the total number of elements currently in the queue. This
|
||||
// estimate is only accurate if the queue has completely stabilized before it is called
|
||||
// (i.e. all enqueue and dequeue operations have completed and their memory effects are
|
||||
// visible on the calling thread, and no further operations start while this method is
|
||||
// being called).
|
||||
// Thread-safe.
|
||||
inline size_t size_approx() const
|
||||
{
|
||||
return (size_t)sema->availableApprox();
|
||||
}
|
||||
|
||||
|
||||
// Returns true if the underlying atomic variables used by
|
||||
// the queue are lock-free (they should be on most platforms).
|
||||
// Thread-safe.
|
||||
static bool is_lock_free()
|
||||
{
|
||||
return ConcurrentQueue::is_lock_free();
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
template<typename U, typename A1, typename A2>
|
||||
static inline U* create(A1&& a1, A2&& a2)
|
||||
{
|
||||
void* p = (Traits::malloc)(sizeof(U));
|
||||
return p != nullptr ? new (p) U(std::forward<A1>(a1), std::forward<A2>(a2)) : nullptr;
|
||||
}
|
||||
|
||||
template<typename U>
|
||||
static inline void destroy(U* p)
|
||||
{
|
||||
if (p != nullptr) {
|
||||
p->~U();
|
||||
}
|
||||
(Traits::free)(p);
|
||||
}
|
||||
|
||||
private:
|
||||
ConcurrentQueue inner;
|
||||
std::unique_ptr<LightweightSemaphore, void (*)(LightweightSemaphore*)> sema;
|
||||
};
|
||||
|
||||
|
||||
template<typename T, typename Traits>
|
||||
inline void swap(BlockingConcurrentQueue<T, Traits>& a, BlockingConcurrentQueue<T, Traits>& b) MOODYCAMEL_NOEXCEPT
|
||||
{
|
||||
a.swap(b);
|
||||
}
|
||||
|
||||
} // end namespace moodycamel
|
||||
|
||||
-3743
File diff suppressed because it is too large.
Load diff
-412
@@ -1,412 +0,0 @@
|
||||
// Provides an efficient implementation of a semaphore (LightweightSemaphore).
|
||||
// This is an extension of Jeff Preshing's sempahore implementation (licensed
|
||||
// under the terms of its separate zlib license) that has been adapted and
|
||||
// extended by Cameron Desrochers.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef> // For std::size_t
|
||||
#include <atomic>
|
||||
#include <type_traits> // For std::make_signed<T>
|
||||
|
||||
#if defined(_WIN32)
|
||||
// Avoid including windows.h in a header; we only need a handful of
|
||||
// items, so we'll redeclare them here (this is relatively safe since
|
||||
// the API generally has to remain stable between Windows versions).
|
||||
// I know this is an ugly hack but it still beats polluting the global
|
||||
// namespace with thousands of generic names or adding a .cpp for nothing.
|
||||
extern "C" {
|
||||
struct _SECURITY_ATTRIBUTES;
|
||||
__declspec(dllimport) void* __stdcall CreateSemaphoreW(_SECURITY_ATTRIBUTES* lpSemaphoreAttributes, long lInitialCount, long lMaximumCount, const wchar_t* lpName);
|
||||
__declspec(dllimport) int __stdcall CloseHandle(void* hObject);
|
||||
__declspec(dllimport) unsigned long __stdcall WaitForSingleObject(void* hHandle, unsigned long dwMilliseconds);
|
||||
__declspec(dllimport) int __stdcall ReleaseSemaphore(void* hSemaphore, long lReleaseCount, long* lpPreviousCount);
|
||||
}
|
||||
#elif defined(__MACH__)
|
||||
#include <mach/mach.h>
|
||||
#elif defined(__unix__)
|
||||
#include <semaphore.h>
|
||||
#endif
|
||||
|
||||
namespace moodycamel
|
||||
{
|
||||
namespace details
|
||||
{
|
||||
|
||||
// Code in the mpmc_sema namespace below is an adaptation of Jeff Preshing's
|
||||
// portable + lightweight semaphore implementations, originally from
|
||||
// https://github.com/preshing/cpp11-on-multicore/blob/master/common/sema.h
|
||||
// LICENSE:
|
||||
// Copyright (c) 2015 Jeff Preshing
|
||||
//
|
||||
// This software is provided 'as-is', without any express or implied
|
||||
// warranty. In no event will the authors be held liable for any damages
|
||||
// arising from the use of this software.
|
||||
//
|
||||
// Permission is granted to anyone to use this software for any purpose,
|
||||
// including commercial applications, and to alter it and redistribute it
|
||||
// freely, subject to the following restrictions:
|
||||
//
|
||||
// 1. The origin of this software must not be misrepresented; you must not
|
||||
// claim that you wrote the original software. If you use this software
|
||||
// in a product, an acknowledgement in the product documentation would be
|
||||
// appreciated but is not required.
|
||||
// 2. Altered source versions must be plainly marked as such, and must not be
|
||||
// misrepresented as being the original software.
|
||||
// 3. This notice may not be removed or altered from any source distribution.
|
||||
#if defined(_WIN32)
|
||||
class Semaphore
|
||||
{
|
||||
private:
|
||||
void* m_hSema;
|
||||
|
||||
Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
|
||||
public:
|
||||
Semaphore(int initialCount = 0)
|
||||
{
|
||||
assert(initialCount >= 0);
|
||||
const long maxLong = 0x7fffffff;
|
||||
m_hSema = CreateSemaphoreW(nullptr, initialCount, maxLong, nullptr);
|
||||
assert(m_hSema);
|
||||
}
|
||||
|
||||
~Semaphore()
|
||||
{
|
||||
CloseHandle(m_hSema);
|
||||
}
|
||||
|
||||
bool wait()
|
||||
{
|
||||
const unsigned long infinite = 0xffffffff;
|
||||
return WaitForSingleObject(m_hSema, infinite) == 0;
|
||||
}
|
||||
|
||||
bool try_wait()
|
||||
{
|
||||
return WaitForSingleObject(m_hSema, 0) == 0;
|
||||
}
|
||||
|
||||
bool timed_wait(std::uint64_t usecs)
|
||||
{
|
||||
return WaitForSingleObject(m_hSema, (unsigned long)(usecs / 1000)) == 0;
|
||||
}
|
||||
|
||||
void signal(int count = 1)
|
||||
{
|
||||
while (!ReleaseSemaphore(m_hSema, count, nullptr));
|
||||
}
|
||||
};
|
||||
#elif defined(__MACH__)
|
||||
//---------------------------------------------------------
|
||||
// Semaphore (Apple iOS and OSX)
|
||||
// Can't use POSIX semaphores due to http://lists.apple.com/archives/darwin-kernel/2009/Apr/msg00010.html
|
||||
//---------------------------------------------------------
|
||||
class Semaphore
|
||||
{
|
||||
private:
|
||||
semaphore_t m_sema;
|
||||
|
||||
Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
|
||||
public:
|
||||
Semaphore(int initialCount = 0)
|
||||
{
|
||||
assert(initialCount >= 0);
|
||||
kern_return_t rc = semaphore_create(mach_task_self(), &m_sema, SYNC_POLICY_FIFO, initialCount);
|
||||
assert(rc == KERN_SUCCESS);
|
||||
(void)rc;
|
||||
}
|
||||
|
||||
~Semaphore()
|
||||
{
|
||||
semaphore_destroy(mach_task_self(), m_sema);
|
||||
}
|
||||
|
||||
bool wait()
|
||||
{
|
||||
return semaphore_wait(m_sema) == KERN_SUCCESS;
|
||||
}
|
||||
|
||||
bool try_wait()
|
||||
{
|
||||
return timed_wait(0);
|
||||
}
|
||||
|
||||
bool timed_wait(std::uint64_t timeout_usecs)
|
||||
{
|
||||
mach_timespec_t ts;
|
||||
ts.tv_sec = static_cast<unsigned int>(timeout_usecs / 1000000);
|
||||
ts.tv_nsec = static_cast<int>((timeout_usecs % 1000000) * 1000);
|
||||
|
||||
// added in OSX 10.10: https://developer.apple.com/library/prerelease/mac/documentation/General/Reference/APIDiffsMacOSX10_10SeedDiff/modules/Darwin.html
|
||||
kern_return_t rc = semaphore_timedwait(m_sema, ts);
|
||||
return rc == KERN_SUCCESS;
|
||||
}
|
||||
|
||||
void signal()
|
||||
{
|
||||
while (semaphore_signal(m_sema) != KERN_SUCCESS);
|
||||
}
|
||||
|
||||
void signal(int count)
|
||||
{
|
||||
while (count-- > 0)
|
||||
{
|
||||
while (semaphore_signal(m_sema) != KERN_SUCCESS);
|
||||
}
|
||||
}
|
||||
};
|
||||
#elif defined(__unix__)
|
||||
//---------------------------------------------------------
|
||||
// Semaphore (POSIX, Linux)
|
||||
//---------------------------------------------------------
|
||||
class Semaphore
|
||||
{
|
||||
private:
|
||||
sem_t m_sema;
|
||||
|
||||
Semaphore(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
Semaphore& operator=(const Semaphore& other) MOODYCAMEL_DELETE_FUNCTION;
|
||||
|
||||
public:
|
||||
Semaphore(int initialCount = 0)
|
||||
{
|
||||
assert(initialCount >= 0);
|
||||
int rc = sem_init(&m_sema, 0, static_cast<unsigned int>(initialCount));
|
||||
assert(rc == 0);
|
||||
(void)rc;
|
||||
}
|
||||
|
||||
~Semaphore()
|
||||
{
|
||||
sem_destroy(&m_sema);
|
||||
}
|
||||
|
||||
bool wait()
|
||||
{
|
||||
// http://stackoverflow.com/questions/2013181/gdb-causes-sem-wait-to-fail-with-eintr-error
|
||||
int rc;
|
||||
do {
|
||||
rc = sem_wait(&m_sema);
|
||||
} while (rc == -1 && errno == EINTR);
|
||||
return rc == 0;
|
||||
}
|
||||
|
||||
bool try_wait()
|
||||
{
|
||||
int rc;
|
||||
do {
|
||||
rc = sem_trywait(&m_sema);
|
||||
} while (rc == -1 && errno == EINTR);
|
||||
return rc == 0;
|
||||
}
|
||||
|
||||
bool timed_wait(std::uint64_t usecs)
|
||||
{
|
||||
struct timespec ts;
|
||||
const int usecs_in_1_sec = 1000000;
|
||||
const int nsecs_in_1_sec = 1000000000;
|
||||
clock_gettime(CLOCK_REALTIME, &ts);
|
||||
ts.tv_sec += (time_t)(usecs / usecs_in_1_sec);
|
||||
ts.tv_nsec += (long)(usecs % usecs_in_1_sec) * 1000;
|
||||
// sem_timedwait bombs if you have more than 1e9 in tv_nsec
|
||||
// so we have to clean things up before passing it in
|
||||
if (ts.tv_nsec >= nsecs_in_1_sec) {
|
||||
ts.tv_nsec -= nsecs_in_1_sec;
|
||||
++ts.tv_sec;
|
||||
}
|
||||
|
||||
int rc;
|
||||
do {
|
||||
rc = sem_timedwait(&m_sema, &ts);
|
||||
} while (rc == -1 && errno == EINTR);
|
||||
return rc == 0;
|
||||
}
|
||||
|
||||
void signal()
|
||||
{
|
||||
while (sem_post(&m_sema) == -1);
|
||||
}
|
||||
|
||||
void signal(int count)
|
||||
{
|
||||
while (count-- > 0)
|
||||
{
|
||||
while (sem_post(&m_sema) == -1);
|
||||
}
|
||||
}
|
||||
};
|
||||
#else
|
||||
#error Unsupported platform! (No semaphore wrapper available)
|
||||
#endif
|
||||
|
||||
} // end namespace details
|
||||
|
||||
|
||||
//---------------------------------------------------------
|
||||
// LightweightSemaphore
|
||||
//---------------------------------------------------------
|
||||
class LightweightSemaphore
|
||||
{
|
||||
public:
|
||||
typedef std::make_signed<std::size_t>::type ssize_t;
|
||||
|
||||
private:
|
||||
std::atomic<ssize_t> m_count;
|
||||
details::Semaphore m_sema;
|
||||
int m_maxSpins;
|
||||
|
||||
bool waitWithPartialSpinning(std::int64_t timeout_usecs = -1)
|
||||
{
|
||||
ssize_t oldCount;
|
||||
int spin = m_maxSpins;
|
||||
while (--spin >= 0)
|
||||
{
|
||||
oldCount = m_count.load(std::memory_order_relaxed);
|
||||
if ((oldCount > 0) && m_count.compare_exchange_strong(oldCount, oldCount - 1, std::memory_order_acquire, std::memory_order_relaxed))
|
||||
return true;
|
||||
std::atomic_signal_fence(std::memory_order_acquire); // Prevent the compiler from collapsing the loop.
|
||||
}
|
||||
oldCount = m_count.fetch_sub(1, std::memory_order_acquire);
|
||||
if (oldCount > 0)
|
||||
return true;
|
||||
if (timeout_usecs < 0)
|
||||
{
|
||||
if (m_sema.wait())
|
||||
return true;
|
||||
}
|
||||
if (timeout_usecs > 0 && m_sema.timed_wait((std::uint64_t)timeout_usecs))
|
||||
return true;
|
||||
// At this point, we've timed out waiting for the semaphore, but the
|
||||
// count is still decremented indicating we may still be waiting on
|
||||
// it. So we have to re-adjust the count, but only if the semaphore
|
||||
// wasn't signaled enough times for us too since then. If it was, we
|
||||
// need to release the semaphore too.
|
||||
while (true)
|
||||
{
|
||||
oldCount = m_count.load(std::memory_order_acquire);
|
||||
if (oldCount >= 0 && m_sema.try_wait())
|
||||
return true;
|
||||
if (oldCount < 0 && m_count.compare_exchange_strong(oldCount, oldCount + 1, std::memory_order_relaxed, std::memory_order_relaxed))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
ssize_t waitManyWithPartialSpinning(ssize_t max, std::int64_t timeout_usecs = -1)
|
||||
{
|
||||
assert(max > 0);
|
||||
ssize_t oldCount;
|
||||
int spin = m_maxSpins;
|
||||
while (--spin >= 0)
|
||||
{
|
||||
oldCount = m_count.load(std::memory_order_relaxed);
|
||||
if (oldCount > 0)
|
||||
{
|
||||
ssize_t newCount = oldCount > max ? oldCount - max : 0;
|
||||
if (m_count.compare_exchange_strong(oldCount, newCount, std::memory_order_acquire, std::memory_order_relaxed))
|
||||
return oldCount - newCount;
|
||||
}
|
||||
std::atomic_signal_fence(std::memory_order_acquire);
|
||||
}
|
||||
oldCount = m_count.fetch_sub(1, std::memory_order_acquire);
|
||||
if (oldCount <= 0)
|
||||
{
|
||||
if ((timeout_usecs == 0) || (timeout_usecs < 0 && !m_sema.wait()) || (timeout_usecs > 0 && !m_sema.timed_wait((std::uint64_t)timeout_usecs)))
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
oldCount = m_count.load(std::memory_order_acquire);
|
||||
if (oldCount >= 0 && m_sema.try_wait())
|
||||
break;
|
||||
if (oldCount < 0 && m_count.compare_exchange_strong(oldCount, oldCount + 1, std::memory_order_relaxed, std::memory_order_relaxed))
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (max > 1)
|
||||
return 1 + tryWaitMany(max - 1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
public:
|
||||
LightweightSemaphore(ssize_t initialCount = 0, int maxSpins = 10000) : m_count(initialCount), m_maxSpins(maxSpins)
|
||||
{
|
||||
assert(initialCount >= 0);
|
||||
assert(maxSpins >= 0);
|
||||
}
|
||||
|
||||
bool tryWait()
|
||||
{
|
||||
ssize_t oldCount = m_count.load(std::memory_order_relaxed);
|
||||
while (oldCount > 0)
|
||||
{
|
||||
if (m_count.compare_exchange_weak(oldCount, oldCount - 1, std::memory_order_acquire, std::memory_order_relaxed))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool wait()
|
||||
{
|
||||
return tryWait() || waitWithPartialSpinning();
|
||||
}
|
||||
|
||||
bool wait(std::int64_t timeout_usecs)
|
||||
{
|
||||
return tryWait() || waitWithPartialSpinning(timeout_usecs);
|
||||
}
|
||||
|
||||
// Acquires between 0 and (greedily) max, inclusive
|
||||
ssize_t tryWaitMany(ssize_t max)
|
||||
{
|
||||
assert(max >= 0);
|
||||
ssize_t oldCount = m_count.load(std::memory_order_relaxed);
|
||||
while (oldCount > 0)
|
||||
{
|
||||
ssize_t newCount = oldCount > max ? oldCount - max : 0;
|
||||
if (m_count.compare_exchange_weak(oldCount, newCount, std::memory_order_acquire, std::memory_order_relaxed))
|
||||
return oldCount - newCount;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Acquires at least one, and (greedily) at most max
|
||||
ssize_t waitMany(ssize_t max, std::int64_t timeout_usecs)
|
||||
{
|
||||
assert(max >= 0);
|
||||
ssize_t result = tryWaitMany(max);
|
||||
if (result == 0 && max > 0)
|
||||
result = waitManyWithPartialSpinning(max, timeout_usecs);
|
||||
return result;
|
||||
}
|
||||
|
||||
ssize_t waitMany(ssize_t max)
|
||||
{
|
||||
ssize_t result = waitMany(max, -1);
|
||||
assert(result > 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
void signal(ssize_t count = 1)
|
||||
{
|
||||
assert(count >= 0);
|
||||
ssize_t oldCount = m_count.fetch_add(count, std::memory_order_release);
|
||||
ssize_t toRelease = -oldCount < count ? -oldCount : count;
|
||||
if (toRelease > 0)
|
||||
{
|
||||
m_sema.signal((int)toRelease);
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t availableApprox() const
|
||||
{
|
||||
ssize_t count = m_count.load(std::memory_order_relaxed);
|
||||
return count > 0 ? static_cast<std::size_t>(count) : 0;
|
||||
}
|
||||
};
|
||||
|
||||
} // end namespace moodycamel
|
||||
|
||||
Vendored
+1
-6
@@ -77,12 +77,7 @@ ifeq ($(USE_SSL),1)
|
||||
endif
|
||||
|
||||
ifeq ($(uname_S),Linux)
|
||||
ifdef OPENSSL_PREFIX
|
||||
CFLAGS+=-I$(OPENSSL_PREFIX)/include
|
||||
SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
|
||||
else
|
||||
SSL_LDFLAGS=-lssl -lcrypto
|
||||
endif
|
||||
SSL_LDFLAGS=-lssl -lcrypto
|
||||
else
|
||||
OPENSSL_PREFIX?=/usr/local/opt/openssl
|
||||
CFLAGS+=-I$(OPENSSL_PREFIX)/include
|
||||
|
||||
Vendored
-1
@@ -174,7 +174,6 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) {
|
||||
return NULL;
|
||||
|
||||
if (elements > 0) {
|
||||
if (SIZE_MAX / sizeof(redisReply*) < elements) return NULL; /* Don't overflow */
|
||||
r->element = hi_calloc(elements,sizeof(redisReply*));
|
||||
if (r->element == NULL) {
|
||||
freeReplyObject(r);
|
||||
|
||||
Vendored
-14
@@ -498,20 +498,6 @@ static void test_reply_reader(void) {
|
||||
freeReplyObject(reply);
|
||||
redisReaderFree(reader);
|
||||
|
||||
test("Multi-bulk never overflows regardless of maxelements: ");
|
||||
size_t bad_mbulk_len = (SIZE_MAX / sizeof(void *)) + 3;
|
||||
char bad_mbulk_reply[100];
|
||||
snprintf(bad_mbulk_reply, sizeof(bad_mbulk_reply), "*%llu\r\n+asdf\r\n",
|
||||
(unsigned long long) bad_mbulk_len);
|
||||
|
||||
reader = redisReaderCreate();
|
||||
reader->maxelements = 0; /* Don't rely on default limit */
|
||||
redisReaderFeed(reader, bad_mbulk_reply, strlen(bad_mbulk_reply));
|
||||
ret = redisReaderGetReply(reader,&reply);
|
||||
test_cond(ret == REDIS_ERR && strcasecmp(reader->errstr, "Out of memory") == 0);
|
||||
freeReplyObject(reply);
|
||||
redisReaderFree(reader);
|
||||
|
||||
#if LLONG_MAX > SIZE_MAX
|
||||
test("Set error when array > SIZE_MAX: ");
|
||||
reader = redisReaderCreate();
|
||||
|
||||
Vendored
BIN
Binary file not shown.
Vendored
-3
@@ -1,3 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
bool FValidKey(const char *key, size_t cch);
|
||||
Vendored
BIN
Binary file not shown.
Vendored
-1
Submodule deps/rocksdb deleted from e3169e3ea8.
+5
-52
@@ -184,9 +184,7 @@ tcp-keepalive 300
|
||||
#
|
||||
# tls-client-key-file-pass secret
|
||||
|
||||
# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange,
|
||||
# required by older versions of OpenSSL (<3.0). Newer versions do not require
|
||||
# this configuration and recommend against it.
|
||||
# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
|
||||
#
|
||||
# tls-dh-params-file keydb.dh
|
||||
|
||||
@@ -259,24 +257,6 @@ tcp-keepalive 300
|
||||
#
|
||||
# tls-session-cache-timeout 60
|
||||
|
||||
# Allow the server to monitor the filesystem and rotate out TLS certificates if
|
||||
# they change on disk, defaults to no.
|
||||
#
|
||||
# tls-rotation no
|
||||
|
||||
# Setup a allowlist of allowed Common Names (CNs)/Subject Alternative Names (SANs)
|
||||
# that are allowed to connect to this server. This includes both normal clients as
|
||||
# well as other servers connected for replication/clustering purposes. If nothing is
|
||||
# specified, then no allowlist is used and all certificates are accepted.
|
||||
# Supports IPv4, DNS, RFC822, and URI SAN types.
|
||||
# You can put multiple names on one line as follows:
|
||||
#
|
||||
# tls-allowlist <dns1> <dns2> <dns3> ...
|
||||
#
|
||||
#
|
||||
# This configuration also allows for wildcard characters with glob style formatting
|
||||
# i.e. "*.com" would allow all clients to connect with a CN/SAN that ends with ".com"
|
||||
|
||||
################################# GENERAL #####################################
|
||||
|
||||
# By default KeyDB does not run as a daemon. Use 'yes' if you need it.
|
||||
@@ -2055,36 +2035,9 @@ server-threads 2
|
||||
# bringing up replicas can result in data loss (the first master will win).
|
||||
# active-replica yes
|
||||
|
||||
# KeyDB will attempt to balance clients across threads evenly; However, replica clients
|
||||
# are usually much more expensive than a normal client, and so KeyDB will try to assign
|
||||
# fewer clients to threads with a replica. The weighting factor below is intented to help tune
|
||||
# this behavior. A replica weighting factor of 2 means we treat a replica as the equivalent
|
||||
# of two normal clients. Adjusting this value may improve performance when replication is
|
||||
# used. The best weighting is workload specific - e.g. read heavy workloads should set
|
||||
# this to 1. Very write heavy workloads may benefit from higher numbers.
|
||||
#
|
||||
# By default KeyDB sets this to 2.
|
||||
replica-weighting-factor 2
|
||||
# Enable Pro? KeyDB pro provides support for pro only features
|
||||
# note: you may omit the license key to demo pro features for a limited time
|
||||
# enable-pro [License Key]
|
||||
|
||||
# Should KeyDB make active attempts at balancing clients across threads? This can impact
|
||||
# performance accepting new clients. By default this is enabled. If disabled there is still
|
||||
# a best effort from the kernel to distribute across threads with SO_REUSEPORT but it will not
|
||||
# be as fair.
|
||||
#
|
||||
# By default this is enabled
|
||||
#
|
||||
active-client-balancing yes
|
||||
|
||||
# Enable FLASH support (Experimental Feature)
|
||||
# Enable FLASH support? (Pro Only)
|
||||
# storage-provider flash /path/to/flash/db
|
||||
|
||||
# Blob support is a way to store very large objects (>200MB) on disk
|
||||
# The files are automatically cleaned up when KeyDB exits and are only
|
||||
# for temporary use. This helps reduce memory pressure for very large
|
||||
# data items at the cost of some performance.
|
||||
#
|
||||
# By default this config is disable. When enabled the disk associated
|
||||
# with KeyDB's working directory will be used. If there is insufficient
|
||||
# disk space or any other I/O error KeyDB will instead use memory.
|
||||
#
|
||||
# blob-support false
|
||||
@@ -1,283 +0,0 @@
|
||||
import random
|
||||
import time
|
||||
import socket
|
||||
import asyncore
|
||||
import threading
|
||||
import argparse
|
||||
from pprint import pprint
|
||||
|
||||
# Globals
|
||||
ops = {}
|
||||
numclients = 0
|
||||
numkeys = 0
|
||||
runtime = 0
|
||||
clients = []
|
||||
|
||||
def _buildResp(*args):
|
||||
result = "*" + str(len(args)) + "\r\n"
|
||||
for v in args:
|
||||
result = result + "$" + str(len(v)) + "\r\n"
|
||||
result = result + v + "\r\n"
|
||||
return result.encode('utf-8')
|
||||
|
||||
class Client(asyncore.dispatcher):
|
||||
def __init__(self, host, port):
|
||||
asyncore.dispatcher.__init__(self)
|
||||
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.connect((host, port))
|
||||
self.buf = b''
|
||||
self.inbuf = b''
|
||||
self.callbacks = list()
|
||||
self.client_id = 0
|
||||
self.get_client_id()
|
||||
|
||||
def handle_connect(self):
|
||||
pass
|
||||
|
||||
def handle_read(self):
|
||||
self.inbuf += self.recv(8192)
|
||||
self.parse_response()
|
||||
|
||||
def handle_write(self):
|
||||
sent = self.send(self.buf)
|
||||
self.buf = self.buf[sent:]
|
||||
|
||||
def handle_close(self):
|
||||
self.close()
|
||||
|
||||
def writable(self):
|
||||
return len(self.buf) > 0
|
||||
|
||||
def parse_array(self, startpos):
|
||||
assert(self.inbuf[startpos] == ord('*'))
|
||||
endrange = self.inbuf[startpos+1:].find(ord('\r')) + 1 + startpos
|
||||
assert(endrange > 0)
|
||||
numargs = int(self.inbuf[startpos+1:endrange])
|
||||
if numargs == -1: # Nil array, used in some returns
|
||||
startpos = endrange + 2
|
||||
return startpos, []
|
||||
assert(numargs > 0)
|
||||
args = list()
|
||||
startpos = endrange + 2 # plus 1 gets us to the '\n' and the next gets us to the start char
|
||||
|
||||
while len(args) < numargs:
|
||||
# We're parsing entries of the form "$N\r\nnnnnnn\r\n"
|
||||
if startpos >= len(self.inbuf):
|
||||
return # Not the full response
|
||||
if self.inbuf[startpos] == ord('*'):
|
||||
startpos, arr = self.parse_array(startpos)
|
||||
args.append(arr)
|
||||
else:
|
||||
assert(self.inbuf[startpos] == ord('$'))
|
||||
startpos = startpos + 1
|
||||
endrange = self.inbuf[startpos:].find(b'\r')
|
||||
if endrange < 0:
|
||||
return
|
||||
endrange += startpos
|
||||
assert(endrange <= len(self.inbuf))
|
||||
length = int(self.inbuf[startpos:endrange])
|
||||
if length < 0:
|
||||
return
|
||||
startpos = endrange + 2
|
||||
assert((startpos + length) <= len(self.inbuf))
|
||||
assert(self.inbuf[startpos+length] == ord('\r'))
|
||||
assert(self.inbuf[startpos+length+1] == ord('\n'))
|
||||
args.append(self.inbuf[startpos:(startpos+length)])
|
||||
startpos += length + 2
|
||||
assert(len(args) == numargs)
|
||||
return startpos, args
|
||||
|
||||
def parse_response(self):
|
||||
if len(self.inbuf) == 0:
|
||||
return
|
||||
|
||||
while len(self.inbuf) > 0:
|
||||
if self.inbuf[0] == ord('+') or self.inbuf[0] == ord('-') or self.inbuf[0] == ord(':'):
|
||||
# This is a single line response
|
||||
endpos = self.inbuf.find(b'\n')
|
||||
if endpos < 0:
|
||||
return # incomplete response
|
||||
self.callbacks[0](self, self.inbuf[0:endpos-1])
|
||||
self.callbacks.pop(0)
|
||||
self.inbuf = self.inbuf[endpos+1:]
|
||||
|
||||
elif self.inbuf[0] == ord('*'):
|
||||
#RESP response
|
||||
try:
|
||||
startpos, args = self.parse_array(0)
|
||||
except:
|
||||
return # Not all data here yet
|
||||
self.callbacks[0](self, args)
|
||||
self.callbacks.pop(0)
|
||||
self.inbuf = self.inbuf[startpos:]
|
||||
else:
|
||||
print("ERROR: Unknown response:")
|
||||
pprint(self.inbuf)
|
||||
assert(False)
|
||||
|
||||
|
||||
def default_result_handler(self, result):
|
||||
pprint(result)
|
||||
|
||||
# Public Methods
|
||||
def set(self, key, val, callback = default_result_handler):
|
||||
self.buf += _buildResp("set", key, val)
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def lpush(self, key, val, callback = default_result_handler):
|
||||
self.buf += _buildResp("lpush", key, val)
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def blpop(self, *keys, timeout=0, callback=default_result_handler):
|
||||
self.buf += _buildResp("blpop", *keys, str(timeout))
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def delete(self, key, callback = default_result_handler):
|
||||
self.buf += _buildResp("del", key)
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def unblock(self, client_id, callback=default_result_handler):
|
||||
self.buf += _buildResp("client", "unblock", str(client_id))
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def scan(self, iter, match=None, count=None, callback = default_result_handler):
|
||||
args = ["scan", str(iter)]
|
||||
if match != None:
|
||||
args.append("MATCH")
|
||||
args.append(match)
|
||||
if count != None:
|
||||
args.append("COUNT")
|
||||
args.append(str(count))
|
||||
self.buf += _buildResp(*args)
|
||||
self.callbacks.append(callback)
|
||||
|
||||
def get_client_id(self):
|
||||
self.buf += _buildResp("client", "id")
|
||||
self.callbacks.append(self.store_client_id)
|
||||
|
||||
def store_client_id(self, c, resp):
|
||||
assert(resp[0] == ord(':'))
|
||||
self.client_id = int(resp[1:])
|
||||
assert(self.client_id == c.client_id)
|
||||
|
||||
def get(self, key, callback = None):
|
||||
return
|
||||
|
||||
def getrandomkey():
|
||||
return str(random.randrange(0, numkeys))
|
||||
|
||||
def handle_lpush_response(c, resp=None):
|
||||
global ops
|
||||
if resp != None:
|
||||
ops['lpush'] += 1
|
||||
assert(resp[0] == ord(':'))
|
||||
c.lpush("list_" + getrandomkey(), 'bardsklfjkldsjfdlsjflksdfjklsdjflksd kldsjflksd jlkdsjf lksdjklds jrfklsdjfklsdjfkl', handle_lpush_response)
|
||||
|
||||
def handle_blpop_response(c, resp=None):
|
||||
global ops
|
||||
if resp != None:
|
||||
ops['blpop'] += 1
|
||||
c.blpop("list_" + getrandomkey(), callback=handle_blpop_response)
|
||||
|
||||
def handle_set_response(c, resp=None):
|
||||
global ops
|
||||
if resp != None:
|
||||
ops['set'] += 1
|
||||
assert(resp[0] == ord('+'))
|
||||
c.set("str_" + getrandomkey(), 'bardsklfjkldsjfdlsjflksdfjklsdjflksd kldsjflksd jlkdsjf lksdjklds jrfklsdjfklsdjfkl', handle_set_response)
|
||||
|
||||
def handle_del_response(c, resp=None):
|
||||
global ops
|
||||
if resp != None:
|
||||
ops['del'] += 1
|
||||
c.delete("list_" + getrandomkey(), handle_del_response)
|
||||
|
||||
def scan_callback(c, resp=None):
|
||||
global ops
|
||||
nextstart = int(resp[0])
|
||||
c.scan(nextstart, count=500, callback=scan_callback)
|
||||
ops['scan'] += 1
|
||||
|
||||
def clear_ops():
|
||||
global ops
|
||||
ops = {'lpush': 0, 'blpop': 0, 'del': 0, 'scan': 0, 'set': 0, 'get': 0}
|
||||
|
||||
def stats_thread():
|
||||
global ops
|
||||
global runtime
|
||||
i = 0
|
||||
while i < runtime or not runtime:
|
||||
time.sleep(1)
|
||||
print("Ops per second: " + str({k:v for (k,v) in ops.items() if v}))
|
||||
clear_ops()
|
||||
i += 1
|
||||
asyncore.close_all()
|
||||
|
||||
def flush_db_sync():
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.connect(('127.0.0.1', 6379))
|
||||
server.send(_buildResp("flushdb"))
|
||||
resp = server.recv(8192)
|
||||
assert(resp[:3] == "+OK".encode('utf-8'))
|
||||
|
||||
def init_blocking():
|
||||
global clients
|
||||
if numkeys > 100 * numclients:
|
||||
print("WARNING: High ratio of keys to clients. Most lpushes will not be popped and unblocking will take a long time!")
|
||||
for i in range(numclients):
|
||||
clients.append(Client('127.0.0.1', 6379))
|
||||
if i % 2:
|
||||
handle_blpop_response(clients[-1])
|
||||
else:
|
||||
handle_lpush_response(clients[-1])
|
||||
|
||||
def init_lpush():
|
||||
global clients
|
||||
for i in range(numclients):
|
||||
clients.append(Client('127.0.0.1', 6379))
|
||||
for i in range (10):
|
||||
handle_lpush_response(clients[-1])
|
||||
#handle_set_response(clients[-1], None)
|
||||
|
||||
scan_client = Client('127.0.0.1', 6379)
|
||||
scan_client.scan(0, count=500, callback=scan_callback)
|
||||
|
||||
del_client = Client('127.0.0.1', 6379)
|
||||
handle_del_response(del_client)
|
||||
|
||||
def main(test, flush):
|
||||
clear_ops()
|
||||
|
||||
if flush:
|
||||
flush_db_sync()
|
||||
|
||||
try:
|
||||
globals()[f"init_{test}"]()
|
||||
except KeyError:
|
||||
print(f"Test \"{test}\" not found. Exiting...")
|
||||
exit()
|
||||
except ConnectionRefusedError:
|
||||
print("Could not connect to server. Is it running?")
|
||||
print("Exiting...")
|
||||
exit()
|
||||
|
||||
threading.Thread(target=stats_thread).start()
|
||||
asyncore.loop()
|
||||
print("Done.")
|
||||
|
||||
parser = argparse.ArgumentParser(description="Test use cases for KeyDB.")
|
||||
parser.add_argument('test', choices=[x[5:] for x in filter(lambda name: name.startswith("init_"), globals().keys())], help="which test to run")
|
||||
parser.add_argument('-c', '--clients', type=int, default=50, help="number of running clients to use")
|
||||
parser.add_argument('-k', '--keys', type=int, default=100000, help="number of keys to choose from for random tests")
|
||||
parser.add_argument('-t', '--runtime', type=int, default=0, help="how long to run the test for (default: 0 for infinite)")
|
||||
parser.add_argument('-f', '--flush', action="store_true", help="flush the db before running the test")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
args = parser.parse_args()
|
||||
except:
|
||||
exit()
|
||||
numclients = args.clients
|
||||
numkeys = args.keys
|
||||
runtime = args.runtime
|
||||
main(args.test, args.flush)
|
||||
+2
-20
@@ -1,21 +1,3 @@
|
||||
### KeyDB DEB Package Source Builds
|
||||
### KeyDB Packaging
|
||||
|
||||
This directory contains scripts and components needed to generate debian packages on different distributions/architectures from source
|
||||
|
||||
The 'debian' directory contains debian source code for bionic, buster and later distributions as it uses functions only available with debhelper11+. 'debian_dh9' is used for xenial, stretch and earlier distributions using debhelper9.
|
||||
|
||||
You will need to install pbuilder `sudo apt install pbuilder` along with other distribution specific dependencies
|
||||
|
||||
Generate deb packages with the following script command run from this directory:
|
||||
|
||||
```
|
||||
$ ./deb-buildsource.sh
|
||||
```
|
||||
|
||||
This generates a directory structure, .dsc file, original.tar.gz, and new changelog for the distribution and architecture installed.
|
||||
|
||||
When complete the produced debian packages will be located in deb_files_generated directory.
|
||||
|
||||
Arguments for the deb-buildsource.sh script that can be passed are either 'None' or '"your custom comments"'. 'None' assumes a permanent changelog entry has been made listing the correct build for the version. If you quote and enter your own comment string, it will be appended to the changelog for that deb package. If no argument is passed a default comment is generated saying that this deb package was generated by this script.
|
||||
|
||||
This script has been tested on xenial/bionic/focal/stretch/buster/bullseye
|
||||
The following directories contain packaging code used to create rpm and deb packages as well as docker images.
|
||||
+8
-4
@@ -1,17 +1,21 @@
|
||||
### KeyDB DEB Package Source Builds
|
||||
|
||||
This directory contains scripts and components needed to generate debian packages on different distributions/architectures using source
|
||||
This directory contains scripts and components needed to generate debian packages on different distributions/architectures from source
|
||||
|
||||
You will need to install pbuilder `sudo apt install pbuilder`
|
||||
The 'debian' directory contains debian source code for bionic, buster and later distributions as it uses functions only available with debhelper11+. 'debian_dh9' is used for xenial, stretch and earlier distributions using debhelper9.
|
||||
|
||||
You will need to install pbuilder `sudo apt install pbuilder` along with other distribution specific dependancies
|
||||
|
||||
Generate deb packages with the following script command run from this directory:
|
||||
|
||||
```
|
||||
$ ./deb-buildsource.sh
|
||||
```
|
||||
This generates a directory structure, .dsc file, original.tar.gz, .changes files and new changelog for the distribution and architecture installed.
|
||||
|
||||
This generates a directory structure, .dsc file, original.tar.gz, and new changelog for the distribution and architecture installed.
|
||||
|
||||
When complete the produced debian packages will be located in deb_files_generated directory.
|
||||
|
||||
Arguments for the deb-buildsource.sh script that can be passed are either 'None' or '"your custom comments"'. 'None' assumes a permanent changelog entry has been made listing the correct build for the version. If you quote and enter your own comment string, it will be appended to the changelog for that deb package. If no argument is passed a default comment is generated saying that this deb package was generated by this script.
|
||||
|
||||
This script has been tested on xenial/bionic/focal/stretch/buster/bullseye
|
||||
This script has been tested on xenial/bionic/stretch/buster
|
||||
+238
-245
@@ -32,17 +32,8 @@
|
||||
# If instead you are interested in using includes to override configuration
|
||||
# options, it is better to use include as the last line.
|
||||
#
|
||||
# Included paths may contain wildcards. All files matching the wildcards will
|
||||
# be included in alphabetical order.
|
||||
# Note that if an include path contains a wildcards but no files match it when
|
||||
# the server is started, the include statement will be ignored and no error will
|
||||
# be emitted. It is safe, therefore, to include wildcard files from empty
|
||||
# directories.
|
||||
#
|
||||
# include /path/to/local.conf
|
||||
# include /path/to/other.conf
|
||||
# include /path/to/fragments/*.conf
|
||||
#
|
||||
|
||||
################################## MODULES #####################################
|
||||
|
||||
@@ -58,7 +49,7 @@
|
||||
# for connections from all available network interfaces on the host machine.
|
||||
# It is possible to listen to just one or multiple selected interfaces using
|
||||
# the "bind" configuration directive, followed by one or more IP addresses.
|
||||
# Each address can be prefixed by "-", which means that redis will not fail to
|
||||
# Each address can be prefixed by "-", which means that KeyDB will not fail to
|
||||
# start if the address is not available. Being not available only refers to
|
||||
# addresses that does not correspond to any network interfece. Addresses that
|
||||
# are already in use will always fail, and unsupported protocols will always BE
|
||||
@@ -74,16 +65,14 @@
|
||||
# internet, binding to all the interfaces is dangerous and will expose the
|
||||
# instance to everybody on the internet. So by default we uncomment the
|
||||
# following bind directive, that will force KeyDB to listen only on the
|
||||
# IPv4 and IPv6 (if available) loopback interface addresses (this means KeyDB will only be able to
|
||||
# accept client connections from the same host that it is running on).
|
||||
# IPv4 and IPv6 (if available) loopback interface addresses (this means KeyDB
|
||||
# will only be able to accept client connections from the same host that it is
|
||||
# running on).
|
||||
#
|
||||
# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
|
||||
# JUST COMMENT OUT THE FOLLOWING LINE.
|
||||
#
|
||||
# You will also need to set a password unless you explicitly disable protected
|
||||
# mode.
|
||||
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
bind 127.0.0.1 -::1
|
||||
bind 127.0.0.1 ::1
|
||||
|
||||
# Protected mode is a layer of security protection, in order to avoid that
|
||||
# KeyDB instances left open on the internet are accessed and exploited.
|
||||
@@ -123,7 +112,7 @@ tcp-backlog 511
|
||||
# incoming connections. There is no default, so KeyDB will not listen
|
||||
# on a unix socket when not specified.
|
||||
#
|
||||
# unixsocket /tmp/keydb.sock
|
||||
# unixsocket /run/keydb.sock
|
||||
# unixsocketperm 700
|
||||
|
||||
# Close the connection after a client is idle for N seconds (0 to disable)
|
||||
@@ -142,8 +131,7 @@ timeout 0
|
||||
# Note that to close the connection the double of the time is needed.
|
||||
# On other kernels the period depends on the kernel configuration.
|
||||
#
|
||||
# A reasonable value for this option is 300 seconds, which is the new
|
||||
# KeyDB default starting with KeyDB 3.2.1.
|
||||
# A reasonable value for this option is 300 seconds, which is the default.
|
||||
tcp-keepalive 300
|
||||
|
||||
################################# TLS/SSL #####################################
|
||||
@@ -217,9 +205,12 @@ tcp-keepalive 300
|
||||
#
|
||||
# tls-cluster yes
|
||||
|
||||
# Explicitly specify TLS versions to support. Allowed values are case insensitive
|
||||
# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or
|
||||
# any combination. To enable only TLSv1.2 and TLSv1.3, use:
|
||||
# By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended
|
||||
# that older formally deprecated versions are kept disabled to reduce the attack surface.
|
||||
# You can explicitly specify TLS versions to support.
|
||||
# Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2",
|
||||
# "TLSv1.3" (OpenSSL >= 1.1.1) or any combination.
|
||||
# To enable only TLSv1.2 and TLSv1.3, use:
|
||||
#
|
||||
# tls-protocols "TLSv1.2 TLSv1.3"
|
||||
|
||||
@@ -257,29 +248,12 @@ tcp-keepalive 300
|
||||
#
|
||||
# tls-session-cache-timeout 60
|
||||
|
||||
# Allow the server to monitor the filesystem and rotate out TLS certificates if
|
||||
# they change on disk, defaults to no.
|
||||
#
|
||||
# tls-rotation no
|
||||
|
||||
# Setup a allowlist of allowed Common Names (CNs)/Subject Alternative Names (SANs)
|
||||
# that are allowed to connect to this server. This includes both normal clients as
|
||||
# well as other servers connected for replication/clustering purposes. If nothing is
|
||||
# specified, then no allowlist is used and all certificates are accepted.
|
||||
# Supports IPv4, DNS, RFC822, and URI SAN types.
|
||||
# You can put multiple names on one line as follows:
|
||||
#
|
||||
# tls-allowlist <dns1> <dns2> <dns3> ...
|
||||
#
|
||||
#
|
||||
# This configuration also allows for wildcard characters with glob style formatting
|
||||
# i.e. "*.com" would allow all clients to connect with a CN/SAN that ends with ".com"
|
||||
|
||||
################################# GENERAL #####################################
|
||||
|
||||
# By default KeyDB does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that KeyDB will write a pid file in /var/run/keydb.pid when daemonized.
|
||||
daemonize yes
|
||||
# When KeyDB is supervised by upstart or systemd, this parameter has no impact.
|
||||
daemonize no
|
||||
|
||||
# If you run KeyDB from upstart or systemd, KeyDB can interact with your
|
||||
# supervision tree. Options:
|
||||
@@ -287,11 +261,17 @@ daemonize yes
|
||||
# supervised upstart - signal upstart by putting KeyDB into SIGSTOP mode
|
||||
# requires "expect stop" in your upstart job config
|
||||
# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
|
||||
# on startup, and updating KeyDB status on a regular
|
||||
# basis.
|
||||
# supervised auto - detect upstart or systemd method based on
|
||||
# UPSTART_JOB or NOTIFY_SOCKET environment variables
|
||||
# Note: these supervision methods only signal "process is ready."
|
||||
# They do not enable continuous pings back to your supervisor.
|
||||
# supervised no
|
||||
#
|
||||
# The default is "no". To run under upstart/systemd, you can simply uncomment
|
||||
# the line below:
|
||||
#
|
||||
# supervised auto
|
||||
|
||||
# If a pid file is specified, KeyDB writes it where specified at startup
|
||||
# and removes it at exit.
|
||||
@@ -302,7 +282,10 @@ daemonize yes
|
||||
#
|
||||
# Creating a pid file is best effort: if KeyDB is not able to create it
|
||||
# nothing bad happens, the server will start and run normally.
|
||||
pidfile /var/run/keydb/keydb-server.pid
|
||||
#
|
||||
# Note that on modern Linux systems "/run/keydb.pid" is more conforming
|
||||
# and should be used instead.
|
||||
pidfile /var/run/keydb_6379.pid
|
||||
|
||||
# Specify the server verbosity level.
|
||||
# This can be one of:
|
||||
@@ -333,7 +316,7 @@ logfile /var/log/keydb/keydb-server.log
|
||||
# crash-log-enabled no
|
||||
|
||||
# To disable the fast memory check that's run as part of the crash log, which
|
||||
# will possibly let keydb terminate sooner, uncomment the following:
|
||||
# will possibly let KeyDB terminate sooner, uncomment the following:
|
||||
#
|
||||
# crash-memcheck-enabled no
|
||||
|
||||
@@ -343,21 +326,19 @@ logfile /var/log/keydb/keydb-server.log
|
||||
databases 16
|
||||
|
||||
# By default KeyDB shows an ASCII art logo only when started to log to the
|
||||
# standard output and if the standard output is a TTY. Basically this means
|
||||
# that normally a logo is displayed only in interactive sessions.
|
||||
# standard output and if the standard output is a TTY and syslog logging is
|
||||
# disabled. Basically this means that normally a logo is displayed only in
|
||||
# interactive sessions.
|
||||
#
|
||||
# However it is possible to force the pre-4.0 behavior and always show a
|
||||
# ASCII art logo in startup logs by setting the following option to yes.
|
||||
always-show-logo yes
|
||||
always-show-logo no
|
||||
|
||||
# By default, KeyDB modifies the process title (as seen in 'top' and 'ps') to
|
||||
# provide some runtime information. It is possible to disable this and leave
|
||||
# the process name as executed by setting the following to no.
|
||||
set-proc-title yes
|
||||
|
||||
# Retrieving "message of today" using CURL requests.
|
||||
#enable-motd yes
|
||||
|
||||
# When changing the process title, KeyDB uses the following template to construct
|
||||
# the modified title.
|
||||
#
|
||||
@@ -376,28 +357,29 @@ set-proc-title yes
|
||||
proc-title-template "{title} {listen-addr} {server-mode}"
|
||||
|
||||
################################ SNAPSHOTTING ################################
|
||||
#
|
||||
# Save the DB on disk:
|
||||
#
|
||||
# save <seconds> <changes>
|
||||
#
|
||||
# Will save the DB if both the given number of seconds and the given
|
||||
# number of write operations against the DB occurred.
|
||||
#
|
||||
# In the example below the behavior will be to save:
|
||||
# after 900 sec (15 min) if at least 1 key changed
|
||||
# after 300 sec (5 min) if at least 10 keys changed
|
||||
# after 60 sec if at least 10000 keys changed
|
||||
#
|
||||
# It is also possible to remove all the previously configured save
|
||||
# points by adding a save directive with a single empty string argument
|
||||
# like in the following example:
|
||||
#
|
||||
# save ""
|
||||
|
||||
save 900 1
|
||||
save 300 10
|
||||
save 60 10000
|
||||
# Save the DB to disk.
|
||||
#
|
||||
# save <seconds> <changes>
|
||||
#
|
||||
# KeyDB will save the DB if both the given number of seconds and the given
|
||||
# number of write operations against the DB occurred.
|
||||
#
|
||||
# Snapshotting can be completely disabled with a single empty string argument
|
||||
# as in following example:
|
||||
#
|
||||
# save ""
|
||||
#
|
||||
# Unless specified otherwise, by default KeyDB will save the DB:
|
||||
# * After 3600 seconds (an hour) if at least 1 key changed
|
||||
# * After 300 seconds (5 minutes) if at least 100 keys changed
|
||||
# * After 60 seconds if at least 10000 keys changed
|
||||
#
|
||||
# You can set these explicitly by uncommenting the three following lines.
|
||||
#
|
||||
# save 3600 1
|
||||
# save 300 100
|
||||
# save 60 10000
|
||||
|
||||
# By default KeyDB will stop accepting writes if RDB snapshots are enabled
|
||||
# (at least one save point) and the latest background save failed.
|
||||
@@ -502,9 +484,9 @@ dir /var/lib/keydb
|
||||
#
|
||||
# However this is not enough if you are using KeyDB ACLs (for KeyDB version
|
||||
# 6 or greater), and the default user is not capable of running the PSYNC
|
||||
# command and/or other commands needed for replication (gathered in the
|
||||
# @replication group). In this case it's better to configure a special user to
|
||||
# use with replication, and specify the masteruser configuration as such:
|
||||
# command and/or other commands needed for replication. In this case it's
|
||||
# better to configure a special user to use with replication, and specify the
|
||||
# masteruser configuration as such:
|
||||
#
|
||||
# masteruser <username>
|
||||
#
|
||||
@@ -526,20 +508,6 @@ dir /var/lib/keydb
|
||||
#
|
||||
replica-serve-stale-data yes
|
||||
|
||||
# Active Replicas will allow read only data access while loading remote RDBs
|
||||
# provided they are permitted to serve stale data. As an option you may also
|
||||
# permit them to accept write commands. This is an EXPERIMENTAL feature and
|
||||
# may result in commands not being fully synchronized
|
||||
#
|
||||
# allow-write-during-load no
|
||||
|
||||
# You can modify the number of masters necessary to form a replica quorum when
|
||||
# multi-master is enabled and replica-serve-stale-data is "no". By default
|
||||
# this is set to -1 which implies the number of known masters (e.g. those
|
||||
# you added with replicaof)
|
||||
#
|
||||
# replica-quorum -1
|
||||
|
||||
# You can configure a replica instance to accept writes or not. Writing against
|
||||
# a replica instance may be useful to store some ephemeral data (because data
|
||||
# written on a replica will be easily deleted after resync with the master) but
|
||||
@@ -558,11 +526,13 @@ replica-read-only yes
|
||||
|
||||
# Replication SYNC strategy: disk or socket.
|
||||
#
|
||||
# New replicas and reconnecting replicas that are not able to continue the
|
||||
# replication process just receiving differences, need to do what is called a
|
||||
# "full synchronization". An RDB file is transmitted from the master to the
|
||||
# replicas.
|
||||
# -------------------------------------------------------
|
||||
# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY
|
||||
# -------------------------------------------------------
|
||||
#
|
||||
# New replicas and reconnecting replicas that are not able to continue the replication
|
||||
# process just receiving differences, need to do what is called a "full
|
||||
# synchronization". An RDB file is transmitted from the master to the replicas.
|
||||
# The transmission can happen in two different ways:
|
||||
#
|
||||
# 1) Disk-backed: The KeyDB master creates a new process that writes the RDB
|
||||
@@ -572,14 +542,14 @@ replica-read-only yes
|
||||
# RDB file to replica sockets, without touching the disk at all.
|
||||
#
|
||||
# With disk-backed replication, while the RDB file is generated, more replicas
|
||||
# can be queued and served with the RDB file as soon as the current child
|
||||
# producing the RDB file finishes its work. With diskless replication instead
|
||||
# once the transfer starts, new replicas arriving will be queued and a new
|
||||
# transfer will start when the current one terminates.
|
||||
# can be queued and served with the RDB file as soon as the current child producing
|
||||
# the RDB file finishes its work. With diskless replication instead once
|
||||
# the transfer starts, new replicas arriving will be queued and a new transfer
|
||||
# will start when the current one terminates.
|
||||
#
|
||||
# When diskless replication is used, the master waits a configurable amount of
|
||||
# time (in seconds) before starting the transfer in the hope that multiple
|
||||
# replicas will arrive and the transfer can be parallelized.
|
||||
# time (in seconds) before starting the transfer in the hope that multiple replicas
|
||||
# will arrive and the transfer can be parallelized.
|
||||
#
|
||||
# With slow disks and fast (large bandwidth) networks, diskless replication
|
||||
# works better.
|
||||
@@ -590,8 +560,8 @@ repl-diskless-sync no
|
||||
# to the replicas.
|
||||
#
|
||||
# This is important since once the transfer starts, it is not possible to serve
|
||||
# new replicas arriving, that will be queued for the next RDB transfer, so the
|
||||
# server waits a delay in order to let more replicas arrive.
|
||||
# new replicas arriving, that will be queued for the next RDB transfer, so the server
|
||||
# waits a delay in order to let more replicas arrive.
|
||||
#
|
||||
# The delay is specified in seconds, and by default is 5 seconds. To disable
|
||||
# it entirely just set it to 0 seconds and the transfer will start ASAP.
|
||||
@@ -602,7 +572,7 @@ repl-diskless-sync-delay 5
|
||||
# does not immediately store an RDB on disk, it may cause data loss during
|
||||
# failovers. RDB diskless load + KeyDB modules not handling I/O reads may also
|
||||
# cause KeyDB to abort in case of I/O errors during the initial synchronization
|
||||
# stage with the master. Use only if your do what you are doing.
|
||||
# stage with the master. Use only if you know what you are doing.
|
||||
# -----------------------------------------------------------------------------
|
||||
#
|
||||
# Replica can load the RDB it reads from the replication link directly from the
|
||||
@@ -658,10 +628,10 @@ repl-diskless-load disabled
|
||||
repl-disable-tcp-nodelay no
|
||||
|
||||
# Set the replication backlog size. The backlog is a buffer that accumulates
|
||||
# replica data when replicas are disconnected for some time, so that when a
|
||||
# replica wants to reconnect again, often a full resync is not needed, but a
|
||||
# partial resync is enough, just passing the portion of data the replica
|
||||
# missed while disconnected.
|
||||
# replica data when replicas are disconnected for some time, so that when a replica
|
||||
# wants to reconnect again, often a full resync is not needed, but a partial
|
||||
# resync is enough, just passing the portion of data the replica missed while
|
||||
# disconnected.
|
||||
#
|
||||
# The bigger the replication backlog, the longer the replica can endure the
|
||||
# disconnect and later be able to perform a partial resynchronization.
|
||||
@@ -683,13 +653,13 @@ repl-disable-tcp-nodelay no
|
||||
#
|
||||
# repl-backlog-ttl 3600
|
||||
|
||||
# The replica priority is an integer number published by KeyDB in the INFO
|
||||
# output. It is used by KeyDB Sentinel in order to select a replica to promote
|
||||
# into a master if the master is no longer working correctly.
|
||||
# The replica priority is an integer number published by KeyDB in the INFO output.
|
||||
# It is used by KeyDB Sentinel in order to select a replica to promote into a
|
||||
# master if the master is no longer working correctly.
|
||||
#
|
||||
# A replica with a low priority number is considered better for promotion, so
|
||||
# for instance if there are three replicas with priority 10, 100, 25 Sentinel
|
||||
# will pick the one with priority 10, that is the lowest.
|
||||
# for instance if there are three replicas with priority 10, 100, 25 Sentinel will
|
||||
# pick the one with priority 10, that is the lowest.
|
||||
#
|
||||
# However a special priority of 0 marks the replica as not able to perform the
|
||||
# role of master, so a replica with priority of 0 will never be selected by
|
||||
@@ -765,7 +735,7 @@ replica-priority 100
|
||||
|
||||
# KeyDB implements server assisted support for client side caching of values.
|
||||
# This is implemented using an invalidation table that remembers, using
|
||||
# 16 millions of slots, what clients may have certain subsets of keys. In turn
|
||||
# a radix key indexed by key name, what clients have which keys. In turn
|
||||
# this is used in order to send invalidation messages to clients. Please
|
||||
# check this page to understand more about the feature:
|
||||
#
|
||||
@@ -835,7 +805,7 @@ replica-priority 100
|
||||
# -<command> Disallow the execution of that command
|
||||
# +@<category> Allow the execution of all the commands in such category
|
||||
# with valid categories are like @admin, @set, @sortedset, ...
|
||||
# and so forth, see the full list in the server.cpp file where
|
||||
# and so forth, see the full list in the server.c file where
|
||||
# the KeyDB command table is described and defined.
|
||||
# The special category @all means all the commands, but currently
|
||||
# present in the server, and that will be loaded in the future
|
||||
@@ -897,40 +867,6 @@ replica-priority 100
|
||||
#
|
||||
# Basically ACL rules are processed left-to-right.
|
||||
#
|
||||
# The following is a list of command categories and their meanings:
|
||||
# * keyspace - Writing or reading from keys, databases, or their metadata
|
||||
# in a type agnostic way. Includes DEL, RESTORE, DUMP, RENAME, EXISTS, DBSIZE,
|
||||
# KEYS, EXPIRE, TTL, FLUSHALL, etc. Commands that may modify the keyspace,
|
||||
# key or metadata will also have `write` category. Commands that only read
|
||||
# the keyspace, key or metadata will have the `read` category.
|
||||
# * read - Reading from keys (values or metadata). Note that commands that don't
|
||||
# interact with keys, will not have either `read` or `write`.
|
||||
# * write - Writing to keys (values or metadata)
|
||||
# * admin - Administrative commands. Normal applications will never need to use
|
||||
# these. Includes REPLICAOF, CONFIG, DEBUG, SAVE, MONITOR, ACL, SHUTDOWN, etc.
|
||||
# * dangerous - Potentially dangerous (each should be considered with care for
|
||||
# various reasons). This includes FLUSHALL, MIGRATE, RESTORE, SORT, KEYS,
|
||||
# CLIENT, DEBUG, INFO, CONFIG, SAVE, REPLICAOF, etc.
|
||||
# * connection - Commands affecting the connection or other connections.
|
||||
# This includes AUTH, SELECT, COMMAND, CLIENT, ECHO, PING, etc.
|
||||
# * blocking - Potentially blocking the connection until released by another
|
||||
# command.
|
||||
# * fast - Fast O(1) commands. May loop on the number of arguments, but not the
|
||||
# number of elements in the key.
|
||||
# * slow - All commands that are not Fast.
|
||||
# * pubsub - PUBLISH / SUBSCRIBE related
|
||||
# * transaction - WATCH / MULTI / EXEC related commands.
|
||||
# * scripting - Scripting related.
|
||||
# * set - Data type: sets related.
|
||||
# * sortedset - Data type: zsets related.
|
||||
# * list - Data type: lists related.
|
||||
# * hash - Data type: hashes related.
|
||||
# * string - Data type: strings related.
|
||||
# * bitmap - Data type: bitmaps related.
|
||||
# * hyperloglog - Data type: hyperloglog related.
|
||||
# * geo - Data type: geo related.
|
||||
# * stream - Data type: streams related.
|
||||
#
|
||||
# For more information about ACL configuration please refer to
|
||||
# the Redis web site at https://redis.io/topics/acl
|
||||
|
||||
@@ -960,7 +896,7 @@ acllog-max-len 128
|
||||
# AUTH <password> as usually, or more explicitly with AUTH default <password>
|
||||
# if they follow the new protocol: both will work.
|
||||
#
|
||||
# The requirepass is not compatible with aclfile option and the ACL LOAD
|
||||
# The requirepass is not compatable with aclfile option and the ACL LOAD
|
||||
# command, these will cause requirepass to be ignored.
|
||||
#
|
||||
# requirepass foobared
|
||||
@@ -968,7 +904,7 @@ acllog-max-len 128
|
||||
# New users are initialized with restrictive permissions by default, via the
|
||||
# equivalent of this ACL rule 'off resetkeys -@all'. Starting with KeyDB 6.2, it
|
||||
# is possible to manage access to Pub/Sub channels with ACL rules as well. The
|
||||
# default Pub/Sub channels permission if new users is controlled by the
|
||||
# default Pub/Sub channels permission if new users is controlled by the
|
||||
# acl-pubsub-default configuration directive, which accepts one of these values:
|
||||
#
|
||||
# allchannels: grants access to all Pub/Sub channels
|
||||
@@ -1057,13 +993,13 @@ acllog-max-len 128
|
||||
# maxmemory <bytes>
|
||||
|
||||
# MAXMEMORY POLICY: how KeyDB will select what to remove when maxmemory
|
||||
# is reached. You can select one from the following behaviors:
|
||||
# is reached. You can select among five behaviors:
|
||||
#
|
||||
# volatile-lru -> Evict using approximated LRU, only keys with an expire set.
|
||||
# volatile-lru -> Evict using approximated LRU among the keys with an expire set.
|
||||
# allkeys-lru -> Evict any key using approximated LRU.
|
||||
# volatile-lfu -> Evict using approximated LFU, only keys with an expire set.
|
||||
# volatile-lfu -> Evict using approximated LFU among the keys with an expire set.
|
||||
# allkeys-lfu -> Evict any key using approximated LFU.
|
||||
# volatile-random -> Remove a random key having an expire set.
|
||||
# volatile-random -> Remove a random key among the ones with an expire set.
|
||||
# allkeys-random -> Remove a random key, any key.
|
||||
# volatile-ttl -> Remove the key with the nearest expire time (minor TTL)
|
||||
# noeviction -> Don't evict anything, just return an error on write operations.
|
||||
@@ -1074,14 +1010,12 @@ acllog-max-len 128
|
||||
# Both LRU, LFU and volatile-ttl are implemented using approximated
|
||||
# randomized algorithms.
|
||||
#
|
||||
# Note: with any of the above policies, KeyDB will return an error on write
|
||||
# operations, when there are no suitable keys for eviction.
|
||||
#
|
||||
# At the date of writing these commands are: set setnx setex append
|
||||
# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd
|
||||
# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby
|
||||
# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby
|
||||
# getset mset msetnx exec sort
|
||||
# Note: with any of the above policies, when there are no suitable keys for
|
||||
# eviction, KeyDB will return an error on write operations that require
|
||||
# more memory. These are usually commands that create new keys, add data or
|
||||
# modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE,
|
||||
# SORT (due to the STORE argument), and EXEC (if the transaction includes any
|
||||
# command that requires memory).
|
||||
#
|
||||
# The default is:
|
||||
#
|
||||
@@ -1100,7 +1034,7 @@ acllog-max-len 128
|
||||
|
||||
# Eviction processing is designed to function well with the default setting.
|
||||
# If there is an unusually large amount of write traffic, this value may need to
|
||||
# be increased. Decreasing this value may reduce latency at the risk of
|
||||
# be increased. Decreasing this value may reduce latency at the risk of
|
||||
# eviction processing effectiveness
|
||||
# 0 = minimum latency, 10 = default, 100 = process without regard to latency
|
||||
#
|
||||
@@ -1112,17 +1046,17 @@ acllog-max-len 128
|
||||
# DEL commands to the replica as keys evict in the master side.
|
||||
#
|
||||
# This behavior ensures that masters and replicas stay consistent, and is usually
|
||||
# what you want, however if your replica is writable, or you want the replica
|
||||
# to have a different memory setting, and you are sure all the writes performed
|
||||
# to the replica are idempotent, then you may change this default (but be sure
|
||||
# to understand what you are doing).
|
||||
# what you want, however if your replica is writable, or you want the replica to have
|
||||
# a different memory setting, and you are sure all the writes performed to the
|
||||
# replica are idempotent, then you may change this default (but be sure to understand
|
||||
# what you are doing).
|
||||
#
|
||||
# Note that since the replica by default does not evict, it may end using more
|
||||
# memory than the one set via maxmemory (there are certain buffers that may
|
||||
# be larger on the replica, or data structures may sometimes take more memory
|
||||
# and so forth). So make sure you monitor your replicas and make sure they
|
||||
# have enough memory to never hit a real out-of-memory condition before the
|
||||
# master hits the configured maxmemory setting.
|
||||
# be larger on the replica, or data structures may sometimes take more memory and so
|
||||
# forth). So make sure you monitor your replicas and make sure they have enough
|
||||
# memory to never hit a real out-of-memory condition before the master hits
|
||||
# the configured maxmemory setting.
|
||||
#
|
||||
# replica-ignore-maxmemory yes
|
||||
|
||||
@@ -1206,6 +1140,52 @@ lazyfree-lazy-user-del no
|
||||
|
||||
lazyfree-lazy-user-flush no
|
||||
|
||||
################################ THREADED I/O #################################
|
||||
|
||||
# KeyDB is mostly single threaded, however there are certain threaded
|
||||
# operations such as UNLINK, slow I/O accesses and other things that are
|
||||
# performed on side threads.
|
||||
#
|
||||
# Now it is also possible to handle KeyDB clients socket reads and writes
|
||||
# in different I/O threads. Since especially writing is so slow, normally
|
||||
# KeyDB users use pipelining in order to speed up the KeyDB performances per
|
||||
# core, and spawn multiple instances in order to scale more. Using I/O
|
||||
# threads it is possible to easily speedup two times KeyDB without resorting
|
||||
# to pipelining nor sharding of the instance.
|
||||
#
|
||||
# By default threading is disabled, we suggest enabling it only in machines
|
||||
# that have at least 4 or more cores, leaving at least one spare core.
|
||||
# Using more than 8 threads is unlikely to help much. We also recommend using
|
||||
# threaded I/O only if you actually have performance problems, with KeyDB
|
||||
# instances being able to use a quite big percentage of CPU time, otherwise
|
||||
# there is no point in using this feature.
|
||||
#
|
||||
# So for instance if you have a four cores boxes, try to use 2 or 3 I/O
|
||||
# threads, if you have a 8 cores, try to use 6 threads. In order to
|
||||
# enable I/O threads use the following configuration directive:
|
||||
#
|
||||
# io-threads 4
|
||||
#
|
||||
# Setting io-threads to 1 will just use the main thread as usual.
|
||||
# When I/O threads are enabled, we only use threads for writes, that is
|
||||
# to thread the write(2) syscall and transfer the client buffers to the
|
||||
# socket. However it is also possible to enable threading of reads and
|
||||
# protocol parsing using the following configuration directive, by setting
|
||||
# it to yes:
|
||||
#
|
||||
# io-threads-do-reads no
|
||||
#
|
||||
# Usually threading reads doesn't help much.
|
||||
#
|
||||
# NOTE 1: This configuration directive cannot be changed at runtime via
|
||||
# CONFIG SET. Aso this feature currently does not work when SSL is
|
||||
# enabled.
|
||||
#
|
||||
# NOTE 2: If you want to test the KeyDB speedup using keydb-benchmark, make
|
||||
# sure you also run the benchmark itself in threaded mode, using the
|
||||
# --threads option to match the number of KeyDB threads, otherwise you'll not
|
||||
# be able to notice the improvements.
|
||||
|
||||
############################ KERNEL OOM CONTROL ##############################
|
||||
|
||||
# On Linux, it is possible to hint the kernel OOM killer on what processes
|
||||
@@ -1243,7 +1223,7 @@ oom-score-adj-values 0 200 800
|
||||
# Usually the kernel Transparent Huge Pages control is set to "madvise" or
|
||||
# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which
|
||||
# case this config has no effect. On systems in which it is set to "always",
|
||||
# KeyDB will attempt to disable it specifically for the KeyDB process in order
|
||||
# KeyDB will attempt to disable it specifically for the keydb process in order
|
||||
# to avoid latency problems specifically with fork(2) and CoW.
|
||||
# If for some reason you prefer to keep it enabled, you can set this config to
|
||||
# "no" and the kernel global to "always".
|
||||
@@ -1268,7 +1248,7 @@ disable-thp yes
|
||||
# If the AOF is enabled on startup KeyDB will load the AOF, that is the file
|
||||
# with the better durability guarantees.
|
||||
#
|
||||
# Please check http://redis.io/topics/persistence for more information.
|
||||
# Please check https://redis.io/topics/persistence for more information.
|
||||
|
||||
appendonly no
|
||||
|
||||
@@ -1398,7 +1378,13 @@ aof-use-rdb-preamble yes
|
||||
lua-time-limit 5000
|
||||
|
||||
################################ KEYDB CLUSTER ###############################
|
||||
|
||||
#
|
||||
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
# WARNING EXPERIMENTAL: KeyDB Cluster is considered to be stable code, however
|
||||
# in order to mark it as "mature" we need to wait for a non trivial percentage
|
||||
# of users to deploy it in production.
|
||||
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
#
|
||||
# Normal KeyDB instances can't be part of a KeyDB Cluster; only nodes that are
|
||||
# started as cluster nodes can. In order to start a KeyDB instance as a
|
||||
# cluster node enable the cluster support uncommenting the following:
|
||||
@@ -1506,7 +1492,7 @@ lua-time-limit 5000
|
||||
# cluster-require-full-coverage yes
|
||||
|
||||
# This option, when set to yes, prevents replicas from trying to failover its
|
||||
# master during master failures. However the master can still perform a
|
||||
# master during master failures. However the replica can still perform a
|
||||
# manual failover, if forced to do so.
|
||||
#
|
||||
# This is useful in different scenarios, especially in the case of multiple
|
||||
@@ -1515,24 +1501,8 @@ lua-time-limit 5000
|
||||
#
|
||||
# cluster-replica-no-failover no
|
||||
|
||||
# This option, when set to yes, allows nodes to serve read traffic while the
|
||||
# the cluster is in a down state, as long as it believes it owns the slots.
|
||||
#
|
||||
# This is useful for two cases. The first case is for when an application
|
||||
# doesn't require consistency of data during node failures or network partitions.
|
||||
# One example of this is a cache, where as long as the node has the data it
|
||||
# should be able to serve it.
|
||||
#
|
||||
# The second use case is for configurations that don't meet the recommended
|
||||
# three shards but want to enable cluster mode and scale later. A
|
||||
# master outage in a 1 or 2 shard configuration causes a read/write outage to the
|
||||
# entire cluster without this option set, with it set there is only a write outage.
|
||||
# Without a quorum of masters, slot ownership will not change automatically.
|
||||
#
|
||||
# cluster-allow-reads-when-down no
|
||||
|
||||
# In order to setup your cluster make sure to read the documentation
|
||||
# available at http://redis.io web site.
|
||||
# available at https://redis.io web site.
|
||||
|
||||
########################## CLUSTER DOCKER/NAT support ########################
|
||||
|
||||
@@ -1550,10 +1520,9 @@ lua-time-limit 5000
|
||||
# * cluster-announce-bus-port
|
||||
#
|
||||
# Each instructs the node about its address, client ports (for connections
|
||||
# without and with TLS), and cluster message
|
||||
# bus port. The information is then published in the header of the bus packets
|
||||
# so that other nodes will be able to correctly map the address of the node
|
||||
# publishing the information.
|
||||
# without and with TLS) and cluster message bus port. The information is then
|
||||
# published in the header of the bus packets so that other nodes will be able to
|
||||
# correctly map the address of the node publishing the information.
|
||||
#
|
||||
# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set
|
||||
# to zero, then cluster-announce-port refers to the TLS port. Note also that
|
||||
@@ -1622,7 +1591,7 @@ latency-monitor-threshold 0
|
||||
############################# EVENT NOTIFICATION ##############################
|
||||
|
||||
# KeyDB can notify Pub/Sub clients about events happening in the key space.
|
||||
# This feature is documented at http://redis.io/topics/notifications
|
||||
# This feature is documented at https://redis.io/topics/notifications
|
||||
#
|
||||
# For instance if keyspace events notification is enabled, and a client
|
||||
# performs a DEL operation on key "foo" stored in the Database 0, two
|
||||
@@ -1670,6 +1639,64 @@ latency-monitor-threshold 0
|
||||
# specify at least one of K or E, no events will be delivered.
|
||||
notify-keyspace-events ""
|
||||
|
||||
############################### GOPHER SERVER #################################
|
||||
|
||||
# KeyDB contains an implementation of the Gopher protocol, as specified in
|
||||
# the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt).
|
||||
#
|
||||
# The Gopher protocol was very popular in the late '90s. It is an alternative
|
||||
# to the web, and the implementation both server and client side is so simple
|
||||
# that the KeyDB server has just 100 lines of code in order to implement this
|
||||
# support.
|
||||
#
|
||||
# What do you do with Gopher nowadays? Well Gopher never *really* died, and
|
||||
# lately there is a movement in order for the Gopher more hierarchical content
|
||||
# composed of just plain text documents to be resurrected. Some want a simpler
|
||||
# internet, others believe that the mainstream internet became too much
|
||||
# controlled, and it's cool to create an alternative space for people that
|
||||
# want a bit of fresh air.
|
||||
#
|
||||
# Anyway for the 10nth birthday of the KeyDB, we gave it the Gopher protocol
|
||||
# as a gift.
|
||||
#
|
||||
# --- HOW IT WORKS? ---
|
||||
#
|
||||
# The KeyDB Gopher support uses the inline protocol of KeyDB, and specifically
|
||||
# two kind of inline requests that were anyway illegal: an empty request
|
||||
# or any request that starts with "/" (there are no KeyDB commands starting
|
||||
# with such a slash). Normal RESP2/RESP3 requests are completely out of the
|
||||
# path of the Gopher protocol implementation and are served as usual as well.
|
||||
#
|
||||
# If you open a connection to KeyDB when Gopher is enabled and send it
|
||||
# a string like "/foo", if there is a key named "/foo" it is served via the
|
||||
# Gopher protocol.
|
||||
#
|
||||
# In order to create a real Gopher "hole" (the name of a Gopher site in Gopher
|
||||
# talking), you likely need a script like the following:
|
||||
#
|
||||
# https://github.com/antirez/gopher2redis
|
||||
#
|
||||
# --- SECURITY WARNING ---
|
||||
#
|
||||
# If you plan to put KeyDB on the internet in a publicly accessible address
|
||||
# to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance.
|
||||
# Once a password is set:
|
||||
#
|
||||
# 1. The Gopher server (when enabled, not by default) will kill serve
|
||||
# content via Gopher.
|
||||
# 2. However other commands cannot be called before the client will
|
||||
# authenticate.
|
||||
#
|
||||
# So use the 'requirepass' option to protect your instance.
|
||||
#
|
||||
# Note that Gopher is not currently supported when 'io-threads-do-reads'
|
||||
# is enabled.
|
||||
#
|
||||
# To enable Gopher support, uncomment the following line and set the option
|
||||
# from no (the default) to yes.
|
||||
#
|
||||
# gopher-enabled no
|
||||
|
||||
############################### ADVANCED CONFIG ###############################
|
||||
|
||||
# Hashes are encoded using a memory efficient data structure when they have a
|
||||
@@ -1742,7 +1769,7 @@ hll-sparse-max-bytes 3000
|
||||
# maximum number of items it may contain before switching to a new node when
|
||||
# appending new stream entries. If any of the following settings are set to
|
||||
# zero, the limit is ignored, so for instance it is possible to set just a
|
||||
# max entires limit by setting max-bytes to 0 and max-entries to the desired
|
||||
# max entries limit by setting max-bytes to 0 and max-entries to the desired
|
||||
# value.
|
||||
stream-node-max-bytes 4096
|
||||
stream-node-max-entries 100
|
||||
@@ -1917,6 +1944,10 @@ rdb-save-incremental-fsync yes
|
||||
|
||||
########################### ACTIVE DEFRAGMENTATION #######################
|
||||
#
|
||||
# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested
|
||||
# even in production and manually tested by multiple engineers for some
|
||||
# time.
|
||||
#
|
||||
# What is active defragmentation?
|
||||
# -------------------------------
|
||||
#
|
||||
@@ -1928,7 +1959,7 @@ rdb-save-incremental-fsync yes
|
||||
# less so with Jemalloc, fortunately) and certain workloads. Normally a server
|
||||
# restart is needed in order to lower the fragmentation, or at least to flush
|
||||
# away all the data and create it again. However thanks to this feature
|
||||
# implemented by Oran Agra for Redis 4.0 this process can happen at runtime
|
||||
# implemented by Oran Agra for KeyDB 4.0 this process can happen at runtime
|
||||
# in a "hot" way, while the server is running.
|
||||
#
|
||||
# Basically when the fragmentation is over a certain level (see the
|
||||
@@ -1956,7 +1987,7 @@ rdb-save-incremental-fsync yes
|
||||
# a good idea to leave the defaults untouched.
|
||||
|
||||
# Enabled active defragmentation
|
||||
# activedefrag no
|
||||
# activedefrag yes
|
||||
|
||||
# Minimum amount of fragmentation waste to start active defrag
|
||||
# active-defrag-ignore-bytes 100mb
|
||||
@@ -1967,13 +1998,11 @@ rdb-save-incremental-fsync yes
|
||||
# Maximum percentage of fragmentation at which we use maximum effort
|
||||
# active-defrag-threshold-upper 100
|
||||
|
||||
# Minimal effort for defrag in CPU percentage, to be used when the lower
|
||||
# threshold is reached
|
||||
# active-defrag-cycle-min 1
|
||||
# Minimal effort for defrag in CPU percentage
|
||||
# active-defrag-cycle-min 5
|
||||
|
||||
# Maximal effort for defrag in CPU percentage, to be used when the upper
|
||||
# threshold is reached
|
||||
# active-defrag-cycle-max 25
|
||||
# Maximal effort for defrag in CPU percentage
|
||||
# active-defrag-cycle-max 75
|
||||
|
||||
# Maximum number of set/hash/zset/list fields that will be processed from
|
||||
# the main dictionary scan
|
||||
@@ -1995,7 +2024,7 @@ jemalloc-bg-thread yes
|
||||
# the bgsave child process. The syntax to specify the cpu list is the same as
|
||||
# the taskset command:
|
||||
#
|
||||
# Set redis server/io threads to cpu affinity 0,2,4,6:
|
||||
# Set KeyDB server/io threads to cpu affinity 0,2,4,6:
|
||||
# server_cpulist 0-7:2
|
||||
#
|
||||
# Set bio threads to cpu affinity 1,3:
|
||||
@@ -2014,23 +2043,6 @@ jemalloc-bg-thread yes
|
||||
#
|
||||
# ignore-warnings ARM64-COW-BUG
|
||||
|
||||
# The minimum number of clients on a thread before KeyDB assigns new connections to a different thread
|
||||
# Tuning this parameter is a tradeoff between locking overhead and distributing the workload over multiple cores
|
||||
# min-clients-per-thread 50
|
||||
|
||||
# How often to run RDB load progress callback?
|
||||
# The callback runs during key load to ping other servers and prevent timeouts.
|
||||
# It also updates load time estimates.
|
||||
# Change these values to run it more or less often. It will run when either condition is true.
|
||||
# Either when x bytes have been processed, or when x keys have been loaded.
|
||||
# loading-process-events-interval-bytes 2097152
|
||||
# loading-process-events-interval-keys 8192
|
||||
|
||||
# Avoid forwarding RREPLAY messages to other masters?
|
||||
# WARNING: This setting is dangerous! You must be certain all masters are connected to each
|
||||
# other in a true mesh topology or data loss will occur!
|
||||
# This command can be used to reduce multimaster bus traffic
|
||||
# multi-master-no-forward no
|
||||
|
||||
# Path to directory for file backed scratchpad. The file backed scratchpad
|
||||
# reduces memory requirements by storing rarely accessed data on disk
|
||||
@@ -2040,8 +2052,6 @@ jemalloc-bg-thread yes
|
||||
# Number of worker threads serving requests. This number should be related to the performance
|
||||
# of your network hardware, not the number of cores on your machine. We don't recommend going
|
||||
# above 4 at this time. By default this is set 1.
|
||||
#
|
||||
# Note: KeyDB does not use io-threads, but io-threads is a config alias for server-threads
|
||||
server-threads 2
|
||||
|
||||
# Should KeyDB pin threads to CPUs? By default this is disabled, and KeyDB will not bind threads.
|
||||
@@ -2053,27 +2063,10 @@ server-threads 2
|
||||
# bringing up replicas can result in data loss (the first master will win).
|
||||
# active-replica yes
|
||||
|
||||
# KeyDB will attempt to balance clients across threads evenly; However, replica clients
|
||||
# are usually much more expensive than a normal client, and so KeyDB will try to assign
|
||||
# fewer clients to threads with a replica. The weighting factor below is intented to help tune
|
||||
# this behavior. A replica weighting factor of 2 means we treat a replica as the equivalent
|
||||
# of two normal clients. Adjusting this value may improve performance when replication is
|
||||
# used. The best weighting is workload specific - e.g. read heavy workloads should set
|
||||
# this to 1. Very write heavy workloads may benefit from higher numbers.
|
||||
#
|
||||
# By default KeyDB sets this to 2.
|
||||
replica-weighting-factor 2
|
||||
# Enable Pro? KeyDB pro provides support for pro only features
|
||||
# note: you may omit the license key to demo pro features for a limited time
|
||||
# enable-pro [License Key]
|
||||
|
||||
# Enable FLASH support (Experimental Feature)
|
||||
# Enable FLASH support? (Pro Only)
|
||||
# storage-provider flash /path/to/flash/db
|
||||
|
||||
# Blob support is a way to store very large objects (>200MB) on disk
|
||||
# The files are automatically cleaned up when KeyDB exits and are only
|
||||
# for temporary use. This helps reduce memory pressure for very large
|
||||
# data items at the cost of some performance.
|
||||
#
|
||||
# By default this config is disable. When enabled the disk associated
|
||||
# with KeyDB's working directory will be used. If there is insufficient
|
||||
# disk space or any other I/O error KeyDB will instead use memory.
|
||||
#
|
||||
# blob-support false
|
||||
+15
-112
@@ -20,12 +20,12 @@
|
||||
# The port that this sentinel instance will run on
|
||||
port 26379
|
||||
|
||||
# By default KeyDB Sentinel does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that KeyDB will write a pid file in /var/run/keydb-sentinel.pid when
|
||||
# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that Redis will write a pid file in /var/run/keydb-sentinel.pid when
|
||||
# daemonized.
|
||||
daemonize yes
|
||||
|
||||
# When running daemonized, KeyDB Sentinel writes a pid file in
|
||||
# When running daemonized, Redis Sentinel writes a pid file in
|
||||
# /var/run/keydb-sentinel.pid by default. You can specify a custom pid file
|
||||
# location here.
|
||||
pidfile /var/run/sentinel/keydb-sentinel.pid
|
||||
@@ -59,7 +59,7 @@ logfile /var/log/keydb/keydb-sentinel.log
|
||||
|
||||
# dir <working-directory>
|
||||
# Every long running process should have a well-defined working directory.
|
||||
# For KeyDB Sentinel to chdir to /tmp at startup is the simplest thing
|
||||
# For Redis Sentinel to chdir to /tmp at startup is the simplest thing
|
||||
# for the process to don't interfere with administrative tasks such as
|
||||
# unmounting filesystems.
|
||||
dir /var/lib/keydb
|
||||
@@ -86,34 +86,22 @@ sentinel monitor mymaster 127.0.0.1 6379 2
|
||||
# sentinel auth-pass <master-name> <password>
|
||||
#
|
||||
# Set the password to use to authenticate with the master and replicas.
|
||||
# Useful if there is a password set in the KeyDB instances to monitor.
|
||||
# Useful if there is a password set in the Redis instances to monitor.
|
||||
#
|
||||
# Note that the master password is also used for replicas, so it is not
|
||||
# possible to set a different password in masters and replicas instances
|
||||
# if you want to be able to monitor these instances with Sentinel.
|
||||
#
|
||||
# However you can have KeyDB instances without the authentication enabled
|
||||
# mixed with KeyDB instances requiring the authentication (as long as the
|
||||
# However you can have Redis instances without the authentication enabled
|
||||
# mixed with Redis instances requiring the authentication (as long as the
|
||||
# password set is the same for all the instances requiring the password) as
|
||||
# the AUTH command will have no effect in KeyDB instances with authentication
|
||||
# the AUTH command will have no effect in Redis instances with authentication
|
||||
# switched off.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel auth-pass mymaster MySUPER--secret-0123passw0rd
|
||||
|
||||
# sentinel auth-user <master-name> <username>
|
||||
#
|
||||
# This is useful in order to authenticate to instances having ACL capabilities,
|
||||
# that is, running KeyDB 6.0 or greater. When just auth-pass is provided the
|
||||
# Sentinel instance will authenticate to KeyDB using the old "AUTH <pass>"
|
||||
# method. When also an username is provided, it will use "AUTH <user> <pass>".
|
||||
# In the KeyDB servers side, the ACL to provide just minimal access to
|
||||
# Sentinel instances, should be configured along the following lines:
|
||||
#
|
||||
# user sentinel-user >somepassword +client +subscribe +publish \
|
||||
# +ping +info +multi +slaveof +config +client +exec on
|
||||
|
||||
# sentinel down-after-milliseconds <master-name> <milliseconds>
|
||||
#
|
||||
# Number of milliseconds the master (or any attached replica or sentinel) should
|
||||
@@ -124,73 +112,6 @@ sentinel monitor mymaster 127.0.0.1 6379 2
|
||||
# Default is 30 seconds.
|
||||
sentinel down-after-milliseconds mymaster 30000
|
||||
|
||||
# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for
|
||||
# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
|
||||
# for more details.
|
||||
|
||||
# Sentinel's ACL users are defined in the following format:
|
||||
#
|
||||
# user <username> ... acl rules ...
|
||||
#
|
||||
# For example:
|
||||
#
|
||||
# user worker +@admin +@connection ~* on >ffa9203c493aa99
|
||||
#
|
||||
# For more information about ACL configuration please refer to the Redis
|
||||
# website at https://redis.io/topics/acl and KeyDB server configuration
|
||||
# template keydb.conf.
|
||||
|
||||
# ACL LOG
|
||||
#
|
||||
# The ACL Log tracks failed commands and authentication events associated
|
||||
# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked
|
||||
# by ACLs. The ACL Log is stored in memory. You can reclaim memory with
|
||||
# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
|
||||
acllog-max-len 128
|
||||
|
||||
# Using an external ACL file
|
||||
#
|
||||
# Instead of configuring users here in this file, it is possible to use
|
||||
# a stand-alone file just listing users. The two methods cannot be mixed:
|
||||
# if you configure users here and at the same time you activate the external
|
||||
# ACL file, the server will refuse to start.
|
||||
#
|
||||
# The format of the external ACL user file is exactly the same as the
|
||||
# format that is used inside keydb.conf to describe users.
|
||||
#
|
||||
# aclfile /etc/keydb/sentinel-users.acl
|
||||
|
||||
# requirepass <password>
|
||||
#
|
||||
# You can configure Sentinel itself to require a password, however when doing
|
||||
# so Sentinel will try to authenticate with the same password to all the
|
||||
# other Sentinels. So you need to configure all your Sentinels in a given
|
||||
# group with the same "requirepass" password. Check the following documentation
|
||||
# for more info: https://redis.io/topics/sentinel
|
||||
#
|
||||
# IMPORTANT NOTE: starting with KeyDB 6.2 "requirepass" is a compatibility
|
||||
# layer on top of the ACL system. The option effect will be just setting
|
||||
# the password for the default user. Clients will still authenticate using
|
||||
# AUTH <password> as usually, or more explicitly with AUTH default <password>
|
||||
# if they follow the new protocol: both will work.
|
||||
#
|
||||
# New config files are advised to use separate authentication control for
|
||||
# incoming connections (via ACL), and for outgoing connections (via
|
||||
# sentinel-user and sentinel-pass)
|
||||
#
|
||||
# The requirepass is not compatable with aclfile option and the ACL LOAD
|
||||
# command, these will cause requirepass to be ignored.
|
||||
|
||||
# sentinel sentinel-user <username>
|
||||
#
|
||||
# You can configure Sentinel to authenticate with other Sentinels with specific
|
||||
# user name.
|
||||
|
||||
# sentinel sentinel-pass <password>
|
||||
#
|
||||
# The password for Sentinel to authenticate with other Sentinels. If sentinel-user
|
||||
# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate.
|
||||
|
||||
# sentinel parallel-syncs <master-name> <numreplicas>
|
||||
#
|
||||
# How many replicas we can reconfigure to point to the new replica simultaneously
|
||||
@@ -251,7 +172,7 @@ sentinel failover-timeout mymaster 180000
|
||||
# generated in the WARNING level (for instance -sdown, -odown, and so forth).
|
||||
# This script should notify the system administrator via email, SMS, or any
|
||||
# other messaging system, that there is something wrong with the monitored
|
||||
# KeyDB systems.
|
||||
# Redis systems.
|
||||
#
|
||||
# The script is called with just two arguments: the first is the event type
|
||||
# and the second the event description.
|
||||
@@ -261,7 +182,7 @@ sentinel failover-timeout mymaster 180000
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel notification-script mymaster /var/keydb/notify.sh
|
||||
# sentinel notification-script mymaster /var/redis/notify.sh
|
||||
|
||||
# CLIENTS RECONFIGURATION SCRIPT
|
||||
#
|
||||
@@ -286,7 +207,7 @@ sentinel failover-timeout mymaster 180000
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel client-reconfig-script mymaster /var/keydb/reconfig.sh
|
||||
# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh
|
||||
|
||||
# SECURITY
|
||||
#
|
||||
@@ -297,11 +218,11 @@ sentinel failover-timeout mymaster 180000
|
||||
|
||||
sentinel deny-scripts-reconfig yes
|
||||
|
||||
# KEYDB COMMANDS RENAMING
|
||||
# REDIS COMMANDS RENAMING
|
||||
#
|
||||
# Sometimes the KeyDB server has certain commands, that are needed for Sentinel
|
||||
# Sometimes the Redis server has certain commands, that are needed for Sentinel
|
||||
# to work correctly, renamed to unguessable strings. This is often the case
|
||||
# of CONFIG and SLAVEOF in the context of providers that provide KeyDB as
|
||||
# of CONFIG and SLAVEOF in the context of providers that provide Redis as
|
||||
# a service, and don't want the customers to reconfigure the instances outside
|
||||
# of the administration console.
|
||||
#
|
||||
@@ -318,24 +239,6 @@ sentinel deny-scripts-reconfig yes
|
||||
# SENTINEL SET can also be used in order to perform this configuration at runtime.
|
||||
#
|
||||
# In order to set a command back to its original name (undo the renaming), it
|
||||
# is possible to just rename a command to itself:
|
||||
# is possible to just rename a command to itsef:
|
||||
#
|
||||
# SENTINEL rename-command mymaster CONFIG CONFIG
|
||||
|
||||
# HOSTNAMES SUPPORT
|
||||
#
|
||||
# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
|
||||
# to specify an IP address. Also, it requires the KeyDB replica-announce-ip
|
||||
# keyword to specify only IP addresses.
|
||||
#
|
||||
# You may enable hostnames support by enabling resolve-hostnames. Note
|
||||
# that you must make sure your DNS is configured properly and that DNS
|
||||
# resolution does not introduce very long delays.
|
||||
#
|
||||
SENTINEL resolve-hostnames no
|
||||
|
||||
# When resolve-hostnames is enabled, Sentinel still uses IP addresses
|
||||
# when exposing instances to users, configuration files, etc. If you want
|
||||
# to retain the hostnames when announced, enable announce-hostnames below.
|
||||
#
|
||||
SENTINEL announce-hostnames no
|
||||
@@ -19,11 +19,6 @@ elif [ "$distributor" == "Ubuntu" ]; then
|
||||
fi
|
||||
codename=$(lsb_release --codename --short)
|
||||
date=$(date +%a," "%d" "%b" "%Y" "%T)
|
||||
|
||||
# overwrite debian bookworm version until updated
|
||||
if [ $codename == "bookworm" ]; then
|
||||
distname=+deb12u1
|
||||
fi
|
||||
pkg_name=keydb-$majorv:$version$distname
|
||||
|
||||
# create build tree
|
||||
@@ -58,9 +53,8 @@ if [ "$codename" == "xenial" ]; then
|
||||
else
|
||||
sudo pbuilder create --distribution $codename
|
||||
fi
|
||||
|
||||
sudo pbuilder --update
|
||||
sudo pbuilder --build *.dsc --logfile /mnt/pbuilderlog.log
|
||||
sudo pbuilder --build *.dsc
|
||||
|
||||
# move new packages to deb_files_generated and clean up
|
||||
cp /var/cache/pbuilder/result/*$version*.deb ../deb_files_generated
|
||||
|
||||
+3
-10
@@ -6,9 +6,7 @@ Build-Depends:
|
||||
debhelper (>= 9~),
|
||||
dpkg-dev (>= 1.17.5),
|
||||
systemd,
|
||||
libsystemd-dev <!nocheck>,
|
||||
procps <!nocheck>,
|
||||
pkg-config <!nocheck>,
|
||||
build-essential <!nocheck>,
|
||||
tcl <!nocheck>,
|
||||
tcl-dev <!nocheck>,
|
||||
@@ -18,16 +16,11 @@ Build-Depends:
|
||||
autotools-dev <!nocheck>,
|
||||
autoconf <!nocheck>,
|
||||
libjemalloc-dev <!nocheck>,
|
||||
libssl-dev <!nocheck>,
|
||||
libsnappy-dev <!nocheck>,
|
||||
zlib1g-dev <!nocheck>,
|
||||
libbz2-dev <!nocheck>,
|
||||
liblz4-dev <!nocheck>,
|
||||
libzstd-dev <!nocheck>
|
||||
libssl-dev <!nocheck>
|
||||
Standards-Version: 4.2.1
|
||||
Homepage: https://docs.keydb.dev/
|
||||
Vcs-Git: https://github.com/Snapchat/KeyDB.git
|
||||
Vcs-Browser: https://github.com/Snapchat/KeyDB
|
||||
Vcs-Git: https://github.com/JohnSully/KeyDB.git
|
||||
Vcs-Browser: https://github.com/JohnSully/KeyDB
|
||||
|
||||
Package: keydb
|
||||
Architecture: all
|
||||
|
||||
@@ -1,96 +1,16 @@
|
||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Contact: John Sully <john@eqalpha.com>
|
||||
Upstream-Name: KeyDB
|
||||
Source: https://github.com/EQ-Alpha/KeyDB
|
||||
Upstream-Name: keydb-pro
|
||||
Source: https://github.com/JohnSully/KeyDB-Pro
|
||||
|
||||
Files: *
|
||||
Copyright:
|
||||
© 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
© 2019 John Sully
|
||||
© 2019-2021 EQ Alpha Technology Ltd.
|
||||
© 2021 Snap Inc.
|
||||
Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
Copyright © 2019, John Sully
|
||||
License: Proprietary
|
||||
|
||||
License: BSD-3-clause
|
||||
|
||||
Files:
|
||||
src/rio.*
|
||||
src/t_zset.c
|
||||
src/ziplist.h
|
||||
src/intset.*
|
||||
src/redis-check-aof.c
|
||||
deps/hiredis/*
|
||||
deps/linenoise/*
|
||||
Copyright:
|
||||
© 2009-2012 Pieter Noordhuis <pcnoordhuis@gmail.com>
|
||||
© 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
License: BSD-3-clause
|
||||
|
||||
Files:
|
||||
src/lzf.h
|
||||
src/lzfP.h
|
||||
src/lzf_d.c
|
||||
src/lzf_c.c
|
||||
Copyright:
|
||||
© 2000-2007 Marc Alexander Lehmann <schmorp@schmorp.de>
|
||||
© 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
License: BSD-2-clause
|
||||
|
||||
Files: src/setproctitle.c
|
||||
Copyright:
|
||||
© 2010 William Ahern
|
||||
© 2013 Salvatore Sanfilippo
|
||||
© 2013 Stam He
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: src/ae_evport.c
|
||||
Copyright: © 2012 Joyent, Inc.
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: src/ae_kqueue.c
|
||||
Copyright: © 2009 Harish Mallipeddi <harish.mallipeddi@gmail.com>
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: utils/install_server.sh
|
||||
Copyright: © 2011 Dvir Volk <dvirsk@gmail.com>
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: deps/jemalloc/*
|
||||
Copyright:
|
||||
© 2002-2012 Jason Evans <jasone@canonware.com>
|
||||
© 2007-2012 Mozilla Foundation
|
||||
© 2009-2012 Facebook, Inc.
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: src/pqsort.*
|
||||
Copyright: © 1992-1993 The Regents of the University of California
|
||||
License: BSD-3-clause
|
||||
|
||||
Files: deps/lua/*
|
||||
Copyright: © 1994-2012 Lua.org, PUC-Ri
|
||||
License: MIT
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
.
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
.
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
Files: pkg/deb/debian/*
|
||||
Copyright:
|
||||
© 2009 Chris Lamb <lamby@debian.org>
|
||||
© 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
|
||||
© 2021 Snap Inc.
|
||||
Files: debian/*
|
||||
Copyright: © 2009 Chris Lamb <lamby@debian.org>
|
||||
License: BSD-3-clause
|
||||
|
||||
License: BSD-2-clause
|
||||
|
||||
@@ -5,4 +5,3 @@ src/keydb-check-rdb /usr/bin
|
||||
src/keydb-cli /usr/bin
|
||||
src/keydb-server /usr/bin
|
||||
src/keydb-sentinel /usr/bin
|
||||
src/keydb-diagnostic-tool /usr/bin
|
||||
@@ -3,8 +3,6 @@
|
||||
include /usr/share/dpkg/buildflags.mk
|
||||
|
||||
export BUILD_TLS=yes
|
||||
export USE_SYSTEMD=yes
|
||||
export ENABLE_FLASH=yes
|
||||
export CFLAGS CPPFLAGS LDFLAGS
|
||||
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
|
||||
export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version=6
|
||||
opts=uversionmangle=s/-?(alpha|beta|rc)/~$1/ \
|
||||
https://github.com/EQ-Alpha/KeyDB/releases .*/archive/(.*).tar.gz
|
||||
https://github.com/JohnSully/KeyDB-Pro/releases .*/archive/(.*).tar.gz
|
||||
@@ -1,53 +0,0 @@
|
||||
#compdef keydb-cli
|
||||
local -a options
|
||||
options=(
|
||||
'-h[Server hostname (default: 127.0.0.1).]: :_hosts'
|
||||
'-p[Server port (default: 6379).]'
|
||||
'-s[Server socket (overrides hostname and port).]'
|
||||
'-a[Password to use when connecting to the server. You can also use the REDISCLI_AUTH environment variable to pass this password more safely (if both are used, this argument takes precedence).]'
|
||||
'--user[Used to send ACL style "AUTH username pass". Needs -a.]'
|
||||
'--pass[Alias of -a for consistency with the new --user option.]'
|
||||
'--askpass[Force user to input password with mask from STDIN. If this argument is used, "-a" and REDISCLI_AUTH environment variable will be ignored.]'
|
||||
'-u[Server URI.]'
|
||||
'-r[Execute specified command N times.]'
|
||||
'-i[When -r is used, waits <interval> seconds per command. It is possible to specify sub-second times like -i 0.1.]'
|
||||
'-n[Database number.]'
|
||||
'-3[Start session in RESP3 protocol mode.]'
|
||||
'-x[Read last argument from STDIN.]'
|
||||
'-d[Delimiter between response bulks for raw formatting (default: \n).]'
|
||||
'-D[D <delimiter> Delimiter between responses for raw formatting (default: \n).]'
|
||||
'-c[Enable cluster mode (follow -ASK and -MOVED redirections).]'
|
||||
'-e[Return exit error code when command execution fails.]'
|
||||
'--raw[Use raw formatting for replies (default when STDOUT is not a tty).]'
|
||||
'--no-raw[Force formatted output even when STDOUT is not a tty.]'
|
||||
'--quoted-input[Force input to be handled as quoted strings.]'
|
||||
'--csv[Output in CSV format.]'
|
||||
'--show-pushes[Whether to print RESP3 PUSH messages. Enabled by default when STDOUT is a tty but can be overriden with --show-pushes no.]'
|
||||
'--stat[Print rolling stats about server: mem, clients, ...]'
|
||||
'--latency[Enter a special mode continuously sampling latency. If you use this mode in an interactive session it runs forever displaying real-time stats. Otherwise if --raw or --csv is specified, or if you redirect the output to a non TTY, it samples the latency for 1 second (you can use -i to change the interval), then produces a single output and exits.]'
|
||||
'--latency-history[Like --latency but tracking latency changes over time. Default time interval is 15 sec. Change it using -i.]'
|
||||
'--latency-dist[Shows latency as a spectrum, requires xterm 256 colors. Default time interval is 1 sec. Change it using -i.]'
|
||||
'--lru-test[Simulate a cache workload with an 80-20 distribution.]'
|
||||
'--replica[Simulate a replica showing commands received from the master.]'
|
||||
'--rdb[Transfer an RDB dump from remote server to local file.]'
|
||||
'--pipe[Transfer raw KeyDB protocol from stdin to server.]'
|
||||
'--pipe-timeout[In --pipe mode, abort with error if after sending all data. no reply is received within <n> seconds. Default timeout: 30. Use 0 to wait forever.]'
|
||||
'--bigkeys[Sample KeyDB keys looking for keys with many elements (complexity).]'
|
||||
'--memkeys[Sample KeyDB keys looking for keys consuming a lot of memory.]'
|
||||
'--memkeys-samples[Sample KeyDB keys looking for keys consuming a lot of memory. And define number of key elements to sample]'
|
||||
'--hotkeys[Sample KeyDB keys looking for hot keys. only works when maxmemory-policy is *lfu.]'
|
||||
'--scan[List all keys using the SCAN command.]'
|
||||
'--pattern[Keys pattern when using the --scan, --bigkeys or --hotkeys options (default: *).]'
|
||||
'--quoted-pattern[Same as --pattern, but the specified string can be quoted, in order to pass an otherwise non binary-safe string.]'
|
||||
'--intrinsic-latency[Run a test to measure intrinsic system latency. The test will run for the specified amount of seconds.]'
|
||||
'--eval[Send an EVAL command using the Lua script at <file>.]'
|
||||
'--ldb[Used with --eval enable the Redis Lua debugger.]'
|
||||
'--ldb-sync-mode[Like --ldb but uses the synchronous Lua debugger, in this mode the server is blocked and script changes are not rolled back from the server memory.]'
|
||||
'--cluster[<command> args... opts... Cluster Manager command and arguments (see below).]'
|
||||
'--verbose[Verbose mode.]'
|
||||
'--no-auth-warning[Dont show warning message when using password on command line interface.]'
|
||||
'--help[Output this help and exit.]'
|
||||
'--version[Output version and exit.]'
|
||||
)
|
||||
|
||||
_arguments -s $options
|
||||
@@ -6,7 +6,6 @@ Build-Depends:
|
||||
debhelper (>= 9~),
|
||||
dpkg-dev (>= 1.17.5),
|
||||
systemd,
|
||||
libsystemd-dev <!nocheck>,
|
||||
procps <!nocheck>,
|
||||
build-essential <!nocheck>,
|
||||
tcl <!nocheck>,
|
||||
@@ -17,16 +16,11 @@ Build-Depends:
|
||||
autotools-dev <!nocheck>,
|
||||
autoconf <!nocheck>,
|
||||
libjemalloc-dev <!nocheck>,
|
||||
libssl-dev <!nocheck>,
|
||||
libsnappy-dev <!nocheck>,
|
||||
zlib1g-dev <!nocheck>,
|
||||
libbz2-dev <!nocheck>,
|
||||
liblz4-dev <!nocheck>,
|
||||
libzstd-dev <!nocheck>
|
||||
libssl-dev <!nocheck>
|
||||
Standards-Version: 4.2.1
|
||||
Homepage: https://docs.keydb.dev/
|
||||
Vcs-Git: https://github.com/Snapchat/KeyDB.git
|
||||
Vcs-Browser: https://github.com/Snapchat/KeyDB
|
||||
Vcs-Git: https://github.com/JohnSully/KeyDB.git
|
||||
Vcs-Browser: https://github.com/JohnSully/KeyDB
|
||||
|
||||
Package: keydb
|
||||
Architecture: all
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Contact: John Sully <john@eqalpha.com>
|
||||
Upstream-Name: KeyDB
|
||||
Source: https://github.com/EQ-Alpha/KeyDB
|
||||
Upstream-Name: keydb
|
||||
Source: https://github.com/JohnSully/KeyDB
|
||||
|
||||
Files: *
|
||||
Copyright:
|
||||
© 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
© 2019 John Sully
|
||||
© 2019-2021 EQ Alpha Technology Ltd.
|
||||
|
||||
Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
|
||||
Copyright © 2019, John Sully
|
||||
License: BSD-3-clause
|
||||
|
||||
Files:
|
||||
@@ -85,10 +82,8 @@ License: MIT
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
Files: pkg/deb/debian_dh9/*
|
||||
Copyright:
|
||||
© 2009 Chris Lamb <lamby@debian.org>
|
||||
© 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
|
||||
Files: debian/*
|
||||
Copyright: © 2009 Chris Lamb <lamby@debian.org>
|
||||
License: BSD-3-clause
|
||||
|
||||
License: BSD-2-clause
|
||||
|
||||
@@ -5,4 +5,3 @@ src/keydb-check-aof /usr/bin
|
||||
src/keydb-check-rdb /usr/bin
|
||||
src/keydb-cli /usr/bin
|
||||
src/keydb-sentinel /usr/bin
|
||||
src/keydb-diagnostic-tool /usr/bin
|
||||
@@ -9,8 +9,6 @@ include /usr/share/dpkg/buildflags.mk
|
||||
#LUA_LDFLAGS = $(addprefix -llua5.1-,$(LUA_LIBS_DEBIAN)) $(addprefix ../deps/lua/src/,$(LUA_OBJECTS))
|
||||
|
||||
export BUILD_TLS=yes
|
||||
export USE_SYSTEMD=yes
|
||||
export ENABLE_FLASH=yes
|
||||
export CFLAGS CPPFLAGS LDFLAGS
|
||||
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
|
||||
export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
version=4
|
||||
opts=uversionmangle=s/-?(alpha|beta|rc)/~$1/ \
|
||||
https://github.com/EQ-Alpha/KeyDB/releases .*/archive/(.*).tar.gz
|
||||
https://github.com/JohnSully/KeyDB/releases .*/archive/(.*).tar.gz
|
||||
@@ -1,94 +1,3 @@
|
||||
keydb (6:6.3.1-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* This point release contains fixes to bugs related to expires, active-rep, and rdb saving
|
||||
* Issues fixed: #419, #422, #428
|
||||
* PRs: #426, #429, #431, #433
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Mon, 23 May 2022 20:00:37 +0000
|
||||
|
||||
keydb (6:6.3.0-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* This release open sources KeyDB Enterprise features into the open source project along with PSYNC for active replication
|
||||
* Partial synchronization for active replication is introduced
|
||||
* MVCC introduced into codebase from KeyDB Enterprise
|
||||
* Async commands added: GET, MGET. These will see perf improvements
|
||||
* KEYS and SCAN commands will no longer be blocking calls
|
||||
* Async Rehash implemented for additional stability to perf
|
||||
* IStorage interface added
|
||||
* In-process background saving (forkless) to comply with maxmemory setting
|
||||
* See v6.3.0 tagged release notes on github for a detailed explanation of these changes
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Wed, 11 May 2022 20:00:37 +0000
|
||||
|
||||
keydb (6:6.2.2-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* Acquire lock in module.cpp to fix module test break
|
||||
* Fix usage of nanosleep() on Apple hardware
|
||||
* Fix networking deadlock (#183)
|
||||
* Remove redundant zfree() in keydb-benchmark
|
||||
* Issues resolved: #378, #384
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Sat, 15 Jan 2022 20:00:37 +0000
|
||||
|
||||
keydb (6:6.2.1-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* This release of KeyDB is at parity with Redis 6.2.6. In addition, the following changes were included:
|
||||
* Systemd support for release packages
|
||||
* Fixes to buffer overflows in the `BITOP SHIFT` and `CRON` commands
|
||||
* Server times are now computed on a seperate thread to improve performance
|
||||
* Now enforces syslog identity and facility as soon as possible
|
||||
* Removed erroneous use of LOG_... flags when using syslog
|
||||
* Fixed erroneous `#endif` leading to build errors on some platforms
|
||||
* Fixed the incorrect counting of client connections
|
||||
* Issues resolved: #355, #370
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Wed, 17 Nov 2021 20:00:37 +0000
|
||||
|
||||
keydb (6:6.2.0-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* Removed unused command line options(-a, -d, -P, -r, -q, --csv, -l, -I, -e, --precision, --cluster, --enable-tracking)
|
||||
* --ms option changed to --time, added --clients, --host, --port, and --threads options
|
||||
* Added keydb-diagnostic-tool
|
||||
* Improved test reliability
|
||||
* Fixed memory leak in keydb-benchmark
|
||||
* Fixed a number of race conditions
|
||||
* Fixed OSX build issues
|
||||
* Feature parity with Redis 6.2.3
|
||||
* Fixed Issues: #323 #325 #276
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Thu, 12 Aug 2021 20:00:37 +0000
|
||||
|
||||
keydb (6:6.0.18-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* 6.0.18 - This release contains a number of improvements to stability and performance.
|
||||
* Additional tests were added with improvements to existing test reliability
|
||||
* Merged Redis 6.0.10 into KeyDB
|
||||
* Issues resolved in this release: #214, #222, #238, #257, #273, #276, #285
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Fri, 26 Mar 2021 16:00:00 +0000
|
||||
|
||||
keydb (6:6.0.16-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* 6.0.16 - This release contains significant performance improvements as well as fixes to the following issues: #234, #236, #233, #207, #229, #231
|
||||
* Reduced memory consumption when Active Replication is not used
|
||||
* Reduced CPU consumption
|
||||
* Improved TLS performance
|
||||
* replica-quorum config for multi-master.Intended to be used with "replica-serve-stale-data no", if at least N replicas are connected we will consider our master link up and serve data.
|
||||
* Subkey expires performance improvement for big sets
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Mon, 28 Sep 2020 16:00:00 +0000
|
||||
|
||||
keydb (6:6.0.13-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* 6.0.13 New features added: KEYDB.MEXISTS, REPLICAOF REMOVE, multi-master-no-forward configuration option
|
||||
* KEYDB.MEXISTS - return exactly which keys exist in the database not just the count
|
||||
* REPLICAOF REMOVE - Remove specific masters in a multi master setup
|
||||
* multi-master-no-forward configuration option - Reduce network traffic in mesh topology multimaster setups
|
||||
* Redis 6.0.5 feature parity
|
||||
* Fixed Issues: 194 Bad directive or wrong number of arguments error on 'lazyfree-lazy-user-del no'; 209 Timeout using modules; 210 Databases are not merged on multi-master sync; 211 rpm packages now signed; 212 rpm file permissions updated
|
||||
|
||||
-- Ben Schermel <ben@eqalpha.com> Mon, 13 Jul 2020 8:00:00 +0000
|
||||
|
||||
keydb (6:6.0.9-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* 6.0.9 Addressing issues 187 - cpu lockup with subkey expire, 190 - missing sentinel binary in keydb-tools.
|
||||
@@ -99,7 +8,7 @@ keydb (6:6.0.8-1distribution_placeholder) codename_placeholder; urgency=medium
|
||||
|
||||
* This is our first release to fully support all Redis 6.0.4 features, including but not limited to: TLS Support (fully supports multithreading!), Client side caching, RESP 3 Support
|
||||
* KeyDB Has also added the following new features: Improved memory efficiency for short strings, Fastlock autotuning, KeyDB.HRENAME (rename a member of a hash)In addition we've spent a lot of time focussing on stability.
|
||||
* The following bug fixes are resolved: 150 - Deadlock in ReplicationFeedMonitors, 170 - KeyDB dying via SIGABORT, 180 - crash after setting maxclients via cmd line, 169 - Write performance of a master dropped sharply when a replica is added
|
||||
* The following bug fixes are resolved: 150 - Deadlock in ReplicationFeedMonitors, 170 - KeyDB dying via SIGABORT, 180 - crash after setting maxclients via cmd line, 169 - Write performance of a master dropped sharply when a slave is added
|
||||
* In addition untracked issues were resolved: Potential deadlock when entering futex sleep, Pub/Sub Async messages may not be sent in a timely manner when load is low, Excessive logging during failed RREPLAY, KeyDB unresponsive handling clients on different threads during RDB load
|
||||
* Updated deb package builds to build from source and phasing out init.d. dh_installsystemd from deb helper 11 used where applicable
|
||||
* Naming conventions are now updated [ keydbpackage_version-build~distribution_architecture] and master changelog included and maintained
|
||||
|
||||
+82
-103
@@ -1,112 +1,91 @@
|
||||
FROM ubuntu:20.04
|
||||
SHELL ["/bin/bash","-c"]
|
||||
FROM ubuntu:18.04
|
||||
# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
|
||||
RUN groupadd -r keydb && useradd -r -g keydb keydb
|
||||
# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases
|
||||
ENV GOSU_VERSION 1.14
|
||||
|
||||
# grab gosu for easy step-down from root
|
||||
# https://github.com/tianon/gosu/releases
|
||||
ENV GOSU_VERSION 1.11
|
||||
RUN set -eux; \
|
||||
savedAptMark="$(apt-mark showmanual)"; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
|
||||
wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
|
||||
export GNUPGHOME="$(mktemp -d)"; \
|
||||
gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
|
||||
gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
|
||||
gpgconf --kill all; \
|
||||
rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
|
||||
apt-mark auto '.*' > /dev/null; \
|
||||
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
|
||||
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
|
||||
chmod +x /usr/local/bin/gosu; \
|
||||
gosu --version; \
|
||||
gosu nobody true
|
||||
# build KeyDB
|
||||
ARG BRANCH
|
||||
RUN set -eux; \
|
||||
\
|
||||
savedAptMark="$(apt-mark showmanual)"; \
|
||||
apt-get update; \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \
|
||||
dpkg-dev \
|
||||
pkg-config \
|
||||
ca-certificates \
|
||||
build-essential \
|
||||
nasm \
|
||||
autotools-dev \
|
||||
autoconf \
|
||||
libjemalloc-dev \
|
||||
tcl \
|
||||
tcl-dev \
|
||||
uuid-dev \
|
||||
libcurl4-openssl-dev \
|
||||
libbz2-dev \
|
||||
libzstd-dev \
|
||||
liblz4-dev \
|
||||
libsnappy-dev \
|
||||
libssl-dev \
|
||||
git; \
|
||||
cd /tmp && git clone --branch $BRANCH https://github.com/Snapchat/KeyDB.git --recursive; \
|
||||
cd /tmp/KeyDB; \
|
||||
# disable protected mode as it relates to docker
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
|
||||
sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
|
||||
make -j$(nproc) BUILD_TLS=yes; \
|
||||
cd src; \
|
||||
strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \
|
||||
mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
|
||||
# clean up unused dependencies
|
||||
echo $savedAptMark; \
|
||||
apt-mark auto '.*' > /dev/null; \
|
||||
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
|
||||
find /usr/local -type f -executable -exec ldd '{}' ';' \
|
||||
| awk '/=>/ { print $(NF-1) }' \
|
||||
| sed 's:.*/::' \
|
||||
| sort -u \
|
||||
| xargs -r dpkg-query --search \
|
||||
| cut -d: -f1 \
|
||||
| sort -u \
|
||||
| xargs -r apt-mark manual \
|
||||
; \
|
||||
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
|
||||
rm -rf /var/lib/apt/lists/*; \
|
||||
# create working directories and organize files
|
||||
# save list of currently installed packages for later so we can clean up
|
||||
savedAptMark="$(apt-mark showmanual)"; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
dirmngr \
|
||||
gnupg \
|
||||
wget \
|
||||
; \
|
||||
\
|
||||
dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \
|
||||
wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \
|
||||
wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \
|
||||
\
|
||||
# verify the signature
|
||||
export GNUPGHOME="$(mktemp -d)"; \
|
||||
gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \
|
||||
gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
|
||||
gpgconf --kill all; \
|
||||
rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
|
||||
\
|
||||
# clean up fetch dependencies
|
||||
apt-mark auto '.*' > /dev/null; \
|
||||
[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
|
||||
apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
|
||||
\
|
||||
chmod +x /usr/local/bin/gosu; \
|
||||
# verify that the binary works
|
||||
gosu --version; \
|
||||
gosu nobody true
|
||||
|
||||
# Load pre-generated equivalent binaries to image (reduces deployment build times)
|
||||
RUN \
|
||||
mkdir /data && chown keydb:keydb /data; \
|
||||
mkdir /flash && chown keydb:keydb /flash; \
|
||||
mkdir -p /etc/keydb; \
|
||||
cp /tmp/KeyDB/keydb.conf /etc/keydb/; \
|
||||
sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
ln -s keydb-cli redis-cli; \
|
||||
cd /etc/keydb; \
|
||||
ln -s keydb.conf redis.conf; \
|
||||
rm -rf /tmp/*
|
||||
# generate entrypoint script
|
||||
mkdir -p /etc/keydb
|
||||
ADD ./app/keydb-* /usr/local/bin/
|
||||
ADD ./app/docker-entrypoint.sh /usr/local/bin/
|
||||
ADD ./app/*.conf /etc/keydb/
|
||||
# Set up config and binaries
|
||||
RUN \
|
||||
cd /usr/local/bin && \
|
||||
sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf && \
|
||||
sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf && \
|
||||
sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf && \
|
||||
sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf && \
|
||||
sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf && \
|
||||
ln -s keydb-cli redis-cli && \
|
||||
cd /etc/keydb && \
|
||||
ln -s keydb.conf redis.conf
|
||||
|
||||
# Ensure deps installed for binaries
|
||||
RUN set -eux; \
|
||||
echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
# set remaining image properties
|
||||
\
|
||||
savedAptMark="$(apt-mark showmanual)"; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
libcurl4 \
|
||||
libc6 \
|
||||
libssl1.1 \
|
||||
libuuid1 \
|
||||
libstdc++6 \
|
||||
libgcc1 \
|
||||
zlib1g \
|
||||
libbz2-1.0 \
|
||||
liblz4-1 \
|
||||
libsnappy1v5 \
|
||||
libzstd1 \
|
||||
; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# create working directories
|
||||
RUN \
|
||||
mkdir /data && chown keydb:keydb /data && \
|
||||
mkdir /flash && chown keydb:keydb /flash
|
||||
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/
|
||||
|
||||
#COPY docker-entrypoint.sh /usr/local/bin/
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
|
||||
# Expose container port and start KeyDB by default on running container
|
||||
EXPOSE 6379
|
||||
CMD ["keydb-server","/etc/keydb/keydb.conf"]
|
||||
CMD ["keydb-server", "/etc/keydb/keydb.conf"]
|
||||
@@ -1,81 +0,0 @@
|
||||
FROM alpine:3.12
|
||||
# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
|
||||
RUN addgroup -S -g 1000 keydb && adduser -S -G keydb -u 999 keydb
|
||||
RUN mkdir -p /etc/keydb
|
||||
ARG BRANCH
|
||||
RUN set -eux; \
|
||||
\
|
||||
apk add --no-cache su-exec; \
|
||||
apk add --no-cache --virtual .build-deps \
|
||||
coreutils \
|
||||
gcc \
|
||||
linux-headers \
|
||||
make \
|
||||
musl-dev \
|
||||
openssl-dev \
|
||||
git \
|
||||
util-linux-dev \
|
||||
curl-dev \
|
||||
g++ \
|
||||
libunwind-dev \
|
||||
bash \
|
||||
perl \
|
||||
git \
|
||||
bzip2-dev \
|
||||
zstd-dev \
|
||||
lz4-dev \
|
||||
snappy-dev \
|
||||
; \
|
||||
cd /tmp && git clone --branch $BRANCH https://github.com/Snapchat/KeyDB.git --recursive; \
|
||||
cd /tmp/KeyDB; \
|
||||
# disable protected mode as it relates to docker
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
|
||||
sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
|
||||
grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
|
||||
make -j$(nproc) BUILD_TLS=yes; \
|
||||
cd src; \
|
||||
strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \
|
||||
mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
|
||||
runDeps="$( \
|
||||
scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
|
||||
| tr ',' '\n' \
|
||||
| sort -u \
|
||||
| awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
|
||||
)"; \
|
||||
apk add --no-network --virtual .keydb-rundeps $runDeps; \
|
||||
apk del --no-network .build-deps; \
|
||||
# create working directories and organize files
|
||||
mkdir /data && chown keydb:keydb /data; \
|
||||
mkdir /flash && chown keydb:keydb /flash; \
|
||||
mkdir -p /etc/keydb; \
|
||||
cp /tmp/KeyDB/keydb.conf /etc/keydb/; \
|
||||
sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
|
||||
sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
|
||||
ln -s keydb-cli redis-cli; \
|
||||
cd /etc/keydb; \
|
||||
ln -s keydb.conf redis.conf; \
|
||||
rm -rf /tmp/*
|
||||
# generate entrypoint script
|
||||
RUN set -eux; \
|
||||
echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo ' exec su-exec keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
|
||||
chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
VOLUME /data
|
||||
WORKDIR /data
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
EXPOSE 6379
|
||||
CMD ["keydb-server", "/etc/keydb/keydb.conf"]
|
||||
@@ -1,7 +1,5 @@
|
||||
This Dockerfile will clone the KeyDB repo, build, and generate a Docker image you can use
|
||||
|
||||
To build, use experimental mode to enable use of build args. Tag the build and specify branch name. The command below will generate your docker image:
|
||||
In order to create a docker image, generate the keydb binaries, copy them to the app directory, copy keydb.conf and sentinel.conf to the app directory as well, then run the following command:
|
||||
|
||||
```
|
||||
DOCKER_CLI_EXPERIMENTAL=enabled docker build --build-arg BRANCH=<keydbBranch> -t <yourImageName>
|
||||
$ sudo docker build . -t <yourimagename>
|
||||
```
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
# first arg is `-f` or `--some-option`
|
||||
# or first arg is `something.conf`
|
||||
if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
|
||||
set -- keydb-server "$@"
|
||||
fi
|
||||
|
||||
# allow the container to be started with `--user`
|
||||
if [ "$1" = 'keydb-server' -a "$(id -u)" = '0' ]; then
|
||||
find . \! -user keydb -exec chown keydb '{}' +
|
||||
exec gosu keydb "$0" "$@"
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
### Generate RPM files for the generated binaries
|
||||
|
||||
After making the binaries you can run the following script
|
||||
After running make to produce keydb binaries you can run the following script to create rpm package
|
||||
|
||||
Usage:
|
||||
```
|
||||
|
||||
@@ -25,13 +25,7 @@ mkdir -p $DIR/keydb_build/keydb_rpm/var/log/keydb
|
||||
|
||||
# move binaries to bin
|
||||
rm $DIR/keydb_build/keydb_rpm/usr/bin/*
|
||||
cp $DIR/../../src/keydb-server $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-sentinel $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-cli $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-benchmark $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-check-aof $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-check-rdb $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-diagnostic-tool $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
cp $DIR/../../src/keydb-* $DIR/keydb_build/keydb_rpm/usr/bin/
|
||||
|
||||
# update spec file with build info
|
||||
sed -i '2d' $DIR/keydb_build/keydb.spec
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
Name : keydb
|
||||
Version : 6.0.5
|
||||
Version : 5.3.3
|
||||
Release : 1%{?dist}
|
||||
Group : Unspecified
|
||||
License : BSD
|
||||
Packager : Snap Inc.
|
||||
Packager : EQ Alpha Technology
|
||||
|
||||
URL : https://keydb.dev
|
||||
Summary : A persistent key-value database
|
||||
|
||||
|
||||
Requires: /bin/awk
|
||||
Requires: logrotate
|
||||
Requires(pre): shadow-utils
|
||||
|
||||
+19
-116
@@ -20,20 +20,20 @@
|
||||
# The port that this sentinel instance will run on
|
||||
port 26379
|
||||
|
||||
# By default KeyDB Sentinel does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that KeyDB will write a pid file in /var/run/keydb-sentinel.pid when
|
||||
# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
|
||||
# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when
|
||||
# daemonized.
|
||||
daemonize no
|
||||
|
||||
# When running daemonized, KeyDB Sentinel writes a pid file in
|
||||
# /var/run/keydb-sentinel.pid by default. You can specify a custom pid file
|
||||
# When running daemonized, Redis Sentinel writes a pid file in
|
||||
# /var/run/redis-sentinel.pid by default. You can specify a custom pid file
|
||||
# location here.
|
||||
pidfile /var/run/sentinel/keydb-sentinel.pid
|
||||
pidfile /var/run/redis-sentinel.pid
|
||||
|
||||
# Specify the log file name. Also the empty string can be used to force
|
||||
# Sentinel to log on the standard output. Note that if you use standard
|
||||
# output for logging but daemonize, logs will be sent to /dev/null
|
||||
logfile /var/log/keydb/keydb-sentinel.log
|
||||
logfile /var/log/redis/sentinel.log
|
||||
|
||||
# sentinel announce-ip <ip>
|
||||
# sentinel announce-port <port>
|
||||
@@ -59,12 +59,12 @@ logfile /var/log/keydb/keydb-sentinel.log
|
||||
|
||||
# dir <working-directory>
|
||||
# Every long running process should have a well-defined working directory.
|
||||
# For KeyDB Sentinel to chdir to /tmp at startup is the simplest thing
|
||||
# For Redis Sentinel to chdir to /tmp at startup is the simplest thing
|
||||
# for the process to don't interfere with administrative tasks such as
|
||||
# unmounting filesystems.
|
||||
dir /tmp
|
||||
|
||||
# sentinel monitor <master-name> <ip> <keydb-port> <quorum>
|
||||
# sentinel monitor <master-name> <ip> <redis-port> <quorum>
|
||||
#
|
||||
# Tells Sentinel to monitor this master, and to consider it in O_DOWN
|
||||
# (Objectively Down) state only if at least <quorum> sentinels agree.
|
||||
@@ -86,34 +86,22 @@ sentinel monitor mymaster 127.0.0.1 6379 2
|
||||
# sentinel auth-pass <master-name> <password>
|
||||
#
|
||||
# Set the password to use to authenticate with the master and replicas.
|
||||
# Useful if there is a password set in the KeyDB instances to monitor.
|
||||
# Useful if there is a password set in the Redis instances to monitor.
|
||||
#
|
||||
# Note that the master password is also used for replicas, so it is not
|
||||
# possible to set a different password in masters and replicas instances
|
||||
# if you want to be able to monitor these instances with Sentinel.
|
||||
#
|
||||
# However you can have KeyDB instances without the authentication enabled
|
||||
# mixed with KeyDB instances requiring the authentication (as long as the
|
||||
# However you can have Redis instances without the authentication enabled
|
||||
# mixed with Redis instances requiring the authentication (as long as the
|
||||
# password set is the same for all the instances requiring the password) as
|
||||
# the AUTH command will have no effect in KeyDB instances with authentication
|
||||
# the AUTH command will have no effect in Redis instances with authentication
|
||||
# switched off.
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel auth-pass mymaster MySUPER--secret-0123passw0rd
|
||||
|
||||
# sentinel auth-user <master-name> <username>
|
||||
#
|
||||
# This is useful in order to authenticate to instances having ACL capabilities,
|
||||
# that is, running KeyDB 6.0 or greater. When just auth-pass is provided the
|
||||
# Sentinel instance will authenticate to KeyDB using the old "AUTH <pass>"
|
||||
# method. When also an username is provided, it will use "AUTH <user> <pass>".
|
||||
# In the KeyDB servers side, the ACL to provide just minimal access to
|
||||
# Sentinel instances, should be configured along the following lines:
|
||||
#
|
||||
# user sentinel-user >somepassword +client +subscribe +publish \
|
||||
# +ping +info +multi +slaveof +config +client +exec on
|
||||
|
||||
# sentinel down-after-milliseconds <master-name> <milliseconds>
|
||||
#
|
||||
# Number of milliseconds the master (or any attached replica or sentinel) should
|
||||
@@ -124,73 +112,6 @@ sentinel monitor mymaster 127.0.0.1 6379 2
|
||||
# Default is 30 seconds.
|
||||
sentinel down-after-milliseconds mymaster 30000
|
||||
|
||||
# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for
|
||||
# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
|
||||
# for more details.
|
||||
|
||||
# Sentinel's ACL users are defined in the following format:
|
||||
#
|
||||
# user <username> ... acl rules ...
|
||||
#
|
||||
# For example:
|
||||
#
|
||||
# user worker +@admin +@connection ~* on >ffa9203c493aa99
|
||||
#
|
||||
# For more information about ACL configuration please refer to the Redis
|
||||
# website at https://redis.io/topics/acl and KeyDB server configuration
|
||||
# template keydb.conf.
|
||||
|
||||
# ACL LOG
|
||||
#
|
||||
# The ACL Log tracks failed commands and authentication events associated
|
||||
# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked
|
||||
# by ACLs. The ACL Log is stored in memory. You can reclaim memory with
|
||||
# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
|
||||
acllog-max-len 128
|
||||
|
||||
# Using an external ACL file
|
||||
#
|
||||
# Instead of configuring users here in this file, it is possible to use
|
||||
# a stand-alone file just listing users. The two methods cannot be mixed:
|
||||
# if you configure users here and at the same time you activate the external
|
||||
# ACL file, the server will refuse to start.
|
||||
#
|
||||
# The format of the external ACL user file is exactly the same as the
|
||||
# format that is used inside keydb.conf to describe users.
|
||||
#
|
||||
# aclfile /etc/keydb/sentinel-users.acl
|
||||
|
||||
# requirepass <password>
|
||||
#
|
||||
# You can configure Sentinel itself to require a password, however when doing
|
||||
# so Sentinel will try to authenticate with the same password to all the
|
||||
# other Sentinels. So you need to configure all your Sentinels in a given
|
||||
# group with the same "requirepass" password. Check the following documentation
|
||||
# for more info: https://redis.io/topics/sentinel
|
||||
#
|
||||
# IMPORTANT NOTE: starting with KeyDB 6.2 "requirepass" is a compatibility
|
||||
# layer on top of the ACL system. The option effect will be just setting
|
||||
# the password for the default user. Clients will still authenticate using
|
||||
# AUTH <password> as usually, or more explicitly with AUTH default <password>
|
||||
# if they follow the new protocol: both will work.
|
||||
#
|
||||
# New config files are advised to use separate authentication control for
|
||||
# incoming connections (via ACL), and for outgoing connections (via
|
||||
# sentinel-user and sentinel-pass)
|
||||
#
|
||||
# The requirepass is not compatable with aclfile option and the ACL LOAD
|
||||
# command, these will cause requirepass to be ignored.
|
||||
|
||||
# sentinel sentinel-user <username>
|
||||
#
|
||||
# You can configure Sentinel to authenticate with other Sentinels with specific
|
||||
# user name.
|
||||
|
||||
# sentinel sentinel-pass <password>
|
||||
#
|
||||
# The password for Sentinel to authenticate with other Sentinels. If sentinel-user
|
||||
# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate.
|
||||
|
||||
# sentinel parallel-syncs <master-name> <numreplicas>
|
||||
#
|
||||
# How many replicas we can reconfigure to point to the new replica simultaneously
|
||||
@@ -251,7 +172,7 @@ sentinel failover-timeout mymaster 180000
|
||||
# generated in the WARNING level (for instance -sdown, -odown, and so forth).
|
||||
# This script should notify the system administrator via email, SMS, or any
|
||||
# other messaging system, that there is something wrong with the monitored
|
||||
# KeyDB systems.
|
||||
# Redis systems.
|
||||
#
|
||||
# The script is called with just two arguments: the first is the event type
|
||||
# and the second the event description.
|
||||
@@ -261,7 +182,7 @@ sentinel failover-timeout mymaster 180000
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel notification-script mymaster /var/keydb/notify.sh
|
||||
# sentinel notification-script mymaster /var/redis/notify.sh
|
||||
|
||||
# CLIENTS RECONFIGURATION SCRIPT
|
||||
#
|
||||
@@ -286,7 +207,7 @@ sentinel failover-timeout mymaster 180000
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# sentinel client-reconfig-script mymaster /var/keydb/reconfig.sh
|
||||
# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh
|
||||
|
||||
# SECURITY
|
||||
#
|
||||
@@ -297,11 +218,11 @@ sentinel failover-timeout mymaster 180000
|
||||
|
||||
sentinel deny-scripts-reconfig yes
|
||||
|
||||
# KEYDB COMMANDS RENAMING
|
||||
# REDIS COMMANDS RENAMING
|
||||
#
|
||||
# Sometimes the KeyDB server has certain commands, that are needed for Sentinel
|
||||
# Sometimes the Redis server has certain commands, that are needed for Sentinel
|
||||
# to work correctly, renamed to unguessable strings. This is often the case
|
||||
# of CONFIG and SLAVEOF in the context of providers that provide KeyDB as
|
||||
# of CONFIG and SLAVEOF in the context of providers that provide Redis as
|
||||
# a service, and don't want the customers to reconfigure the instances outside
|
||||
# of the administration console.
|
||||
#
|
||||
@@ -318,24 +239,6 @@ sentinel deny-scripts-reconfig yes
|
||||
# SENTINEL SET can also be used in order to perform this configuration at runtime.
|
||||
#
|
||||
# In order to set a command back to its original name (undo the renaming), it
|
||||
# is possible to just rename a command to itself:
|
||||
# is possible to just rename a command to itsef:
|
||||
#
|
||||
# SENTINEL rename-command mymaster CONFIG CONFIG
|
||||
|
||||
# HOSTNAMES SUPPORT
|
||||
#
|
||||
# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
|
||||
# to specify an IP address. Also, it requires the KeyDB replica-announce-ip
|
||||
# keyword to specify only IP addresses.
|
||||
#
|
||||
# You may enable hostnames support by enabling resolve-hostnames. Note
|
||||
# that you must make sure your DNS is configured properly and that DNS
|
||||
# resolution does not introduce very long delays.
|
||||
#
|
||||
SENTINEL resolve-hostnames no
|
||||
|
||||
# When resolve-hostnames is enabled, Sentinel still uses IP addresses
|
||||
# when exposing instances to users, configuration files, etc. If you want
|
||||
# to retain the hostnames when announced, enable announce-hostnames below.
|
||||
#
|
||||
SENTINEL announce-hostnames no
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,7 +1,7 @@
|
||||
[Unit]
|
||||
Description=Advanced key-value store
|
||||
After=network.target
|
||||
Documentation=http://keydb.io/documentation, man:keydb-sentinel(1)
|
||||
Documentation=https://docs.keydb.dev, man:keydb-sentinel(1)
|
||||
|
||||
[Service]
|
||||
Type=forking
|
||||
|
||||
Regular → Executable
+1
-4
@@ -1,8 +1,5 @@
|
||||
Copyright (c) 2006-2020, Salvatore Sanfilippo
|
||||
Copyright (c) 2006-2015, Salvatore Sanfilippo
|
||||
Copyright (C) 2019, John Sully
|
||||
Copyright (C) 2020-2021, EQ Alpha Technology Ltd.
|
||||
Copyright (C) 2021, Snap Inc.
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
Regular → Executable
File mode changed.
Regular → Executable
File mode changed.
Regular → Executable
File mode changed.
@@ -16,7 +16,6 @@ fi
|
||||
$MAKE -C tests/modules && \
|
||||
$TCLSH tests/test_helper.tcl \
|
||||
--single unit/moduleapi/commandfilter \
|
||||
--single unit/moduleapi/basics \
|
||||
--single unit/moduleapi/fork \
|
||||
--single unit/moduleapi/testrdb \
|
||||
--single unit/moduleapi/infotest \
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
#!/bin/sh
|
||||
TCL_VERSIONS="8.5 8.6"
|
||||
TCLSH=""
|
||||
|
||||
export ASAN_OPTIONS=allocator_may_return_null=1 $ASAN_OPTIONS
|
||||
|
||||
for VERSION in $TCL_VERSIONS; do
|
||||
TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL
|
||||
done
|
||||
|
||||
if [ -z $TCLSH ]
|
||||
then
|
||||
echo "You need tcl 8.5 or newer in order to run the KeyDB test"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -r tests/tls ] || [ ! -r tests/tls_1 ] || [ ! -r tests/tls_2 ];
|
||||
then
|
||||
echo "Generating neccessary certificates for TLS rotation testing."
|
||||
rm -rf tests/tls tests/tls_1 tests/tls_2
|
||||
|
||||
utils/gen-test-certs.sh
|
||||
mv tests/tls tests/tls_1
|
||||
utils/gen-test-certs.sh
|
||||
mv tests/tls tests/tls_2
|
||||
utils/gen-test-certs.sh
|
||||
fi
|
||||
$TCLSH tests/test_helper.tcl \
|
||||
--single unit/tls-rotation \
|
||||
--tls \
|
||||
--config server-threads 3 \
|
||||
"${@}"
|
||||
@@ -1,131 +0,0 @@
|
||||
#include "AsyncWorkQueue.h"
|
||||
#include "server.h"
|
||||
|
||||
AsyncWorkQueue::AsyncWorkQueue(int nthreads)
|
||||
{
|
||||
for (int i = 0; i < nthreads; ++i)
|
||||
{
|
||||
m_vecthreads.emplace_back([&]{
|
||||
WorkerThreadMain();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void AsyncWorkQueue::WorkerThreadMain()
|
||||
{
|
||||
redisServerThreadVars vars;
|
||||
serverTL = &vars;
|
||||
|
||||
vars.clients_pending_asyncwrite = listCreate();
|
||||
|
||||
m_mutex.lock();
|
||||
m_vecpthreadVars.push_back(&vars);
|
||||
m_mutex.unlock();
|
||||
|
||||
while (!m_fQuitting)
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_mutex);
|
||||
if (m_workqueue.empty())
|
||||
m_cvWakeup.wait(lock);
|
||||
|
||||
aeThreadOnline();
|
||||
while (!m_workqueue.empty())
|
||||
{
|
||||
WorkItem task = std::move(m_workqueue.front());
|
||||
m_workqueue.pop_front();
|
||||
|
||||
lock.unlock();
|
||||
serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch();
|
||||
task.fnAsync();
|
||||
g_pserver->garbageCollector.endEpoch(serverTL->gcEpoch);
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
lock.unlock();
|
||||
serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch();
|
||||
if (listLength(serverTL->clients_pending_asyncwrite)) {
|
||||
aeAcquireLock();
|
||||
ProcessPendingAsyncWrites();
|
||||
aeReleaseLock();
|
||||
}
|
||||
g_pserver->garbageCollector.endEpoch(serverTL->gcEpoch);
|
||||
serverTL->gcEpoch.reset();
|
||||
aeThreadOffline();
|
||||
}
|
||||
|
||||
listRelease(vars.clients_pending_asyncwrite);
|
||||
|
||||
std::unique_lock<fastlock> lockf(serverTL->lockPendingWrite);
|
||||
serverTL->vecclientsProcess.clear();
|
||||
serverTL->clients_pending_write.clear();
|
||||
std::atomic_thread_fence(std::memory_order_seq_cst);
|
||||
}
|
||||
|
||||
bool AsyncWorkQueue::removeClientAsyncWrites(client *c)
|
||||
{
|
||||
bool fFound = false;
|
||||
aeAcquireLock();
|
||||
m_mutex.lock();
|
||||
for (auto pvars : m_vecpthreadVars)
|
||||
{
|
||||
listIter li;
|
||||
listNode *ln;
|
||||
listRewind(pvars->clients_pending_asyncwrite, &li);
|
||||
while ((ln = listNext(&li)) != nullptr)
|
||||
{
|
||||
if (c == listNodeValue(ln))
|
||||
{
|
||||
listDelNode(pvars->clients_pending_asyncwrite, ln);
|
||||
fFound = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
m_mutex.unlock();
|
||||
aeReleaseLock();
|
||||
return fFound;
|
||||
}
|
||||
|
||||
void AsyncWorkQueue::shutdown()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_mutex);
|
||||
serverAssert(!GlobalLocksAcquired());
|
||||
m_fQuitting = true;
|
||||
m_cvWakeup.notify_all();
|
||||
lock.unlock();
|
||||
|
||||
for (auto &thread : m_vecthreads)
|
||||
thread.join();
|
||||
}
|
||||
|
||||
void AsyncWorkQueue::abandonThreads()
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_mutex);
|
||||
m_fQuitting = true;
|
||||
m_cvWakeup.notify_all();
|
||||
for (auto &thread : m_vecthreads)
|
||||
{
|
||||
thread.detach();
|
||||
}
|
||||
m_vecthreads.clear();
|
||||
}
|
||||
|
||||
AsyncWorkQueue::~AsyncWorkQueue()
|
||||
{
|
||||
serverAssert(!GlobalLocksAcquired() || m_vecthreads.empty());
|
||||
std::unique_lock<std::mutex> lock(m_mutex);
|
||||
m_fQuitting = true;
|
||||
m_cvWakeup.notify_all();
|
||||
lock.unlock();
|
||||
|
||||
abandonThreads();
|
||||
}
|
||||
|
||||
void AsyncWorkQueue::AddWorkFunction(std::function<void()> &&fnAsync, bool fHiPri)
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(m_mutex);
|
||||
if (fHiPri)
|
||||
m_workqueue.emplace_front(std::move(fnAsync));
|
||||
else
|
||||
m_workqueue.emplace_back(std::move(fnAsync));
|
||||
m_cvWakeup.notify_one();
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
#pragma once
|
||||
#include "fastlock.h"
|
||||
#include <vector>
|
||||
#include <deque>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <condition_variable>
|
||||
#include <atomic>
|
||||
#include <functional>
|
||||
|
||||
class AsyncWorkQueue
|
||||
{
|
||||
struct WorkItem
|
||||
{
|
||||
WorkItem(std::function<void()> &&fnAsync)
|
||||
: fnAsync(std::move(fnAsync))
|
||||
{}
|
||||
|
||||
WorkItem(WorkItem&&) = default;
|
||||
std::function<void()> fnAsync;
|
||||
};
|
||||
std::vector<std::thread> m_vecthreads;
|
||||
std::vector<struct redisServerThreadVars*> m_vecpthreadVars;
|
||||
std::deque<WorkItem> m_workqueue;
|
||||
std::mutex m_mutex;
|
||||
std::condition_variable m_cvWakeup;
|
||||
std::atomic<bool> m_fQuitting { false };
|
||||
|
||||
void WorkerThreadMain();
|
||||
public:
|
||||
AsyncWorkQueue(int nthreads);
|
||||
~AsyncWorkQueue();
|
||||
|
||||
void AddWorkFunction(std::function<void()> &&fnAsync, bool fHiPri = false);
|
||||
bool removeClientAsyncWrites(struct client *c);
|
||||
|
||||
void shutdown();
|
||||
|
||||
void abandonThreads();
|
||||
};
|
||||
@@ -1,54 +0,0 @@
|
||||
#pragma once
|
||||
#include <functional>
|
||||
#include "sds.h"
|
||||
|
||||
#define METADATA_DB_IDENTIFIER "c299fde0-6d42-4ec4-b939-34f680ffe39f"
|
||||
|
||||
class IStorageFactory
|
||||
{
|
||||
public:
|
||||
typedef void (*key_load_iterator)(const char *rgchKey, size_t cchKey, void *privdata);
|
||||
|
||||
virtual ~IStorageFactory() {}
|
||||
virtual class IStorage *create(int db, key_load_iterator itr, void *privdata) = 0;
|
||||
virtual class IStorage *createMetadataDb() = 0;
|
||||
virtual const char *name() const = 0;
|
||||
virtual size_t totalDiskspaceUsed() const = 0;
|
||||
virtual bool FSlow() const = 0;
|
||||
virtual size_t filedsRequired() const { return 0; }
|
||||
};
|
||||
|
||||
class IStorage
|
||||
{
|
||||
public:
|
||||
typedef std::function<bool(const char *, size_t, const void *, size_t)> callback;
|
||||
typedef std::function<void(const char *, size_t, const void *, size_t)> callbackSingle;
|
||||
|
||||
virtual ~IStorage();
|
||||
|
||||
virtual void insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwire) = 0;
|
||||
virtual bool erase(const char *key, size_t cchKey) = 0;
|
||||
virtual void retrieve(const char *key, size_t cchKey, callbackSingle fn) const = 0;
|
||||
virtual size_t clear() = 0;
|
||||
virtual bool enumerate(callback fn) const = 0;
|
||||
virtual size_t count() const = 0;
|
||||
|
||||
virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) {
|
||||
beginWriteBatch();
|
||||
for (size_t ielem = 0; ielem < celem; ++ielem) {
|
||||
insert(rgkeys[ielem], rgcbkeys[ielem], rgvals[ielem], rgcbvals[ielem], false);
|
||||
}
|
||||
endWriteBatch();
|
||||
}
|
||||
|
||||
virtual void beginWriteBatch() {} // NOP
|
||||
virtual void endWriteBatch() {} // NOP
|
||||
|
||||
virtual void batch_lock() {} // NOP
|
||||
virtual void batch_unlock() {} // NOP
|
||||
|
||||
virtual void flush() = 0;
|
||||
|
||||
/* This is permitted to be a shallow clone */
|
||||
virtual const IStorage *clone() const = 0;
|
||||
};
|
||||
+17
-61
@@ -12,15 +12,11 @@
|
||||
# Dependencies are stored in the Makefile.dep file. To rebuild this file
|
||||
# Just use 'make dep', but this is only needed by developers.
|
||||
|
||||
# we ship KeyDB with TLS by default
|
||||
# export it here as both this file and deps/Makefile uses it
|
||||
export BUILD_TLS ?= yes
|
||||
|
||||
release_hdr := $(shell sh -c './mkreleasehdr.sh')
|
||||
uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
|
||||
uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
|
||||
OPTIMIZATION?=-O2 -flto
|
||||
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram rocksdb
|
||||
OPTIMIZATION?=-O2
|
||||
DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram
|
||||
NODEPS:=clean distclean
|
||||
|
||||
# Default settings
|
||||
@@ -62,7 +58,6 @@ endif
|
||||
endif
|
||||
|
||||
USEASM?=true
|
||||
ENABLE_FLASH?=no
|
||||
|
||||
ifneq ($(strip $(SANITIZE)),)
|
||||
CFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE -fno-omit-frame-pointer
|
||||
@@ -72,23 +67,10 @@ ifneq ($(strip $(SANITIZE)),)
|
||||
USEASM=false
|
||||
endif
|
||||
|
||||
ifeq ($(ENABLE_FLASH),yes)
|
||||
FINAL_LIBS+= -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
|
||||
CXXFLAGS+= -I../deps/rocksdb/include/ -DENABLE_ROCKSDB
|
||||
STORAGE_OBJ+= storage/rocksdb.o storage/rocksdbfactory.o
|
||||
FINAL_LIBS+= ../deps/rocksdb/librocksdb.a
|
||||
DEPENDENCY_TARGETS+= rocksdb
|
||||
endif
|
||||
|
||||
|
||||
ifeq ($(CHECKED),true)
|
||||
CXXFLAGS+= -DCHECKED_BUILD
|
||||
endif
|
||||
|
||||
# Do we use our assembly spinlock? X64 only
|
||||
ifeq ($(uname_S),Linux)
|
||||
ifeq ($(uname_M),x86_64)
|
||||
ifneq ($(TARGET32), true)
|
||||
ifneq ($(@@),32bit)
|
||||
ifeq ($(USEASM),true)
|
||||
ASM_OBJ+= fastlock_x64.o
|
||||
CFLAGS+= -DASM_SPINLOCK
|
||||
@@ -100,6 +82,7 @@ endif
|
||||
|
||||
ifeq ($(COMPILER_NAME),clang)
|
||||
CXXFLAGS+= -stdlib=libc++
|
||||
LDFLAGS+= -latomic
|
||||
endif
|
||||
|
||||
# To get ARM stack traces if KeyDB crashes we need a special C flag.
|
||||
@@ -130,19 +113,14 @@ ifeq ($(USE_JEMALLOC),no)
|
||||
MALLOC=libc
|
||||
endif
|
||||
|
||||
|
||||
ifeq ($(NO_LICENSE_CHECK),yes)
|
||||
CXXFLAGS+=-DNO_LICENSE_CHECK=1
|
||||
endif
|
||||
|
||||
# Override default settings if possible
|
||||
-include .make-settings
|
||||
|
||||
DEBUG=-g -ggdb
|
||||
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
|
||||
FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
|
||||
FINAL_LDFLAGS=$(LDFLAGS) $(KEYDB_LDFLAGS) $(DEBUG)
|
||||
FINAL_LIBS+=-lm -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
|
||||
FINAL_LIBS+=-lm
|
||||
DEBUG=-g -ggdb
|
||||
|
||||
ifneq ($(uname_S),Darwin)
|
||||
FINAL_LIBS+=-latomic
|
||||
@@ -178,10 +156,12 @@ ifeq ($(uname_S),Darwin)
|
||||
# must be referenced explicitly during build.
|
||||
ifeq ($(uname_M),arm64)
|
||||
# Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX
|
||||
OPENSSL_PREFIX?=/opt/homebrew/opt/openssl
|
||||
OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include
|
||||
OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib
|
||||
else
|
||||
# Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX
|
||||
OPENSSL_PREFIX?=/usr/local/opt/openssl
|
||||
OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include
|
||||
OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib
|
||||
endif
|
||||
else
|
||||
ifeq ($(uname_S),AIX)
|
||||
@@ -249,18 +229,9 @@ endif
|
||||
endif
|
||||
endif
|
||||
endif
|
||||
|
||||
ifdef OPENSSL_PREFIX
|
||||
OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include
|
||||
OPENSSL_CXXFLAGS=-I$(OPENSSL_PREFIX)/include
|
||||
OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib
|
||||
# Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles
|
||||
export OPENSSL_PREFIX
|
||||
endif
|
||||
|
||||
# Include paths to dependencies
|
||||
FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
|
||||
FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/rocksdb/include/ -I../deps/concurrentqueue
|
||||
FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
|
||||
|
||||
# Determine systemd support and/or build preference (defaulting to auto-detection)
|
||||
BUILD_WITH_SYSTEMD=no
|
||||
@@ -287,7 +258,6 @@ endif
|
||||
ifeq ($(BUILD_WITH_SYSTEMD),yes)
|
||||
FINAL_LIBS+=$(LIBSYSTEMD_LIBS)
|
||||
FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD
|
||||
FINAL_CXXFLAGS+= -DHAVE_LIBSYSTEMD
|
||||
endif
|
||||
|
||||
ifeq ($(MALLOC),tcmalloc)
|
||||
@@ -369,17 +339,15 @@ ENDCOLOR="\033[0m"
|
||||
|
||||
ifndef V
|
||||
QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR);
|
||||
QUIET_CP = @printf ' %b %b\n' $(CCCOLOR)COPY$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR);
|
||||
QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR);
|
||||
QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR);
|
||||
endif
|
||||
|
||||
REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX)
|
||||
REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX)
|
||||
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd_server.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/rocksdb.o storage/rocksdbfactory.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
|
||||
KEYDB_SERVER_OBJ=SnapshotPayloadParseState.o
|
||||
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o $(ASM_OBJ)
|
||||
REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX)
|
||||
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o motd_client.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
|
||||
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o motd.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
|
||||
REDIS_BENCHMARK_NAME=keydb-benchmark$(PROG_SUFFIX)
|
||||
REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o release.o crcspeed.o crc64.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
|
||||
REDIS_CHECK_RDB_NAME=keydb-check-rdb$(PROG_SUFFIX)
|
||||
@@ -436,8 +404,8 @@ endif
|
||||
@touch $@
|
||||
|
||||
# keydb-server
|
||||
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(KEYDB_SERVER_OBJ)
|
||||
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/rocksdb/librocksdb.a $(FINAL_LIBS)
|
||||
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ)
|
||||
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a $(FINAL_LIBS)
|
||||
|
||||
# keydb-sentinel
|
||||
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
|
||||
@@ -463,18 +431,12 @@ $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)
|
||||
$(KEYDB_DIAGNOSTIC_NAME): $(KEYDB_DIAGNOSTIC_OBJ)
|
||||
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS)
|
||||
|
||||
DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(KEYDB_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
|
||||
DEP = $(REDIS_SERVER_OBJ:%.o=%.d) $(REDIS_CLI_OBJ:%.o=%.d) $(REDIS_BENCHMARK_OBJ:%.o=%.d)
|
||||
-include $(DEP)
|
||||
|
||||
# Because the jemalloc.h header is generated as a part of the jemalloc build,
|
||||
# building it should complete before building any other object. Instead of
|
||||
# depending on a single artifact, build all dependencies first.
|
||||
motd_client.o: motd.cpp .make-prerequisites
|
||||
$(REDIS_CXX) -MMD -o motd_client.o -c $< -DCLIENT -fno-lto
|
||||
|
||||
motd_server.o: motd.cpp .make-prerequisites
|
||||
$(REDIS_CXX) -MMD -o motd_server.o -c $< -DSERVER
|
||||
|
||||
%.o: %.c .make-prerequisites
|
||||
$(REDIS_CC) -MMD -o $@ -c $<
|
||||
|
||||
@@ -486,8 +448,6 @@ motd_server.o: motd.cpp .make-prerequisites
|
||||
|
||||
clean:
|
||||
rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) $(KEYDB_DIAGNOSTIC_NAME) *.o *.gcda *.gcno *.gcov KeyDB.info lcov-html Makefile.dep
|
||||
rm -rf storage/*.o
|
||||
rm -rf keydb-server
|
||||
rm -f $(DEP)
|
||||
|
||||
.PHONY: clean
|
||||
@@ -495,7 +455,6 @@ clean:
|
||||
distclean: clean
|
||||
-(cd ../deps && $(MAKE) distclean)
|
||||
-(cd modules && $(MAKE) clean)
|
||||
-(cd ../tests/modules && $(MAKE) clean)
|
||||
-(rm -f .make-*)
|
||||
|
||||
.PHONY: distclean
|
||||
@@ -503,9 +462,6 @@ distclean: clean
|
||||
test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
|
||||
@(cd ..; ./runtest)
|
||||
|
||||
test-modules: $(REDIS_SERVER_NAME)
|
||||
@(cd ..; ./runtest-moduleapi)
|
||||
|
||||
test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME)
|
||||
@(cd ..; ./runtest-sentinel)
|
||||
|
||||
@@ -527,7 +483,7 @@ bench: $(REDIS_BENCHMARK_NAME)
|
||||
@echo ""
|
||||
@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
|
||||
@echo ""
|
||||
$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32" TARGET32=true
|
||||
$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32"
|
||||
|
||||
gcov:
|
||||
$(MAKE) KEYDB_CXXFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_LDFLAGS="-fprofile-arcs -ftest-coverage"
|
||||
|
||||
@@ -1,344 +0,0 @@
|
||||
#include "server.h"
|
||||
#include "SnapshotPayloadParseState.h"
|
||||
#include <sys/mman.h>
|
||||
|
||||
static const size_t SLAB_SIZE = 2*1024*1024; // Note 64 MB because we try to give 64MB to a block
|
||||
class SlabAllocator
|
||||
{
|
||||
class Slab {
|
||||
char *m_pv = nullptr;
|
||||
size_t m_cb = 0;
|
||||
size_t m_cbAllocated = 0;
|
||||
|
||||
public:
|
||||
Slab(size_t cbAllocate) {
|
||||
m_pv = (char*)zmalloc(cbAllocate);
|
||||
m_cb = cbAllocate;
|
||||
}
|
||||
|
||||
~Slab() {
|
||||
zfree(m_pv);
|
||||
}
|
||||
|
||||
Slab(Slab &&other) {
|
||||
m_pv = other.m_pv;
|
||||
m_cb = other.m_cb;
|
||||
m_cbAllocated = other.m_cbAllocated;
|
||||
other.m_pv = nullptr;
|
||||
other.m_cb = 0;
|
||||
other.m_cbAllocated = 0;
|
||||
}
|
||||
|
||||
bool canAllocate(size_t cb) const {
|
||||
return (m_cbAllocated + cb) <= m_cb;
|
||||
}
|
||||
|
||||
void *allocate(size_t cb) {
|
||||
if ((m_cbAllocated + cb) > m_cb)
|
||||
return nullptr;
|
||||
void *pvret = m_pv + m_cbAllocated;
|
||||
m_cbAllocated += cb;
|
||||
return pvret;
|
||||
}
|
||||
};
|
||||
std::vector<Slab> m_vecslabs;
|
||||
public:
|
||||
|
||||
void *allocate(size_t cb) {
|
||||
if (m_vecslabs.empty() || !m_vecslabs.back().canAllocate(cb)) {
|
||||
m_vecslabs.emplace_back(std::max(cb, SLAB_SIZE));
|
||||
}
|
||||
return m_vecslabs.back().allocate(cb);
|
||||
}
|
||||
};
|
||||
|
||||
static uint64_t dictCStringHash(const void *key) {
|
||||
return dictGenHashFunction((unsigned char*)key, strlen((char*)key));
|
||||
}
|
||||
|
||||
|
||||
static void dictKeyDestructor(void *privdata, void *key)
|
||||
{
|
||||
DICT_NOTUSED(privdata);
|
||||
sdsfree((sds)key);
|
||||
}
|
||||
|
||||
static int dictCStringCompare(void *, const void *key1, const void *key2)
|
||||
{
|
||||
int l1,l2;
|
||||
|
||||
l1 = strlen((sds)key1);
|
||||
l2 = strlen((sds)key2);
|
||||
if (l1 != l2) return 0;
|
||||
return memcmp(key1, key2, l1) == 0;
|
||||
}
|
||||
|
||||
dictType metadataLongLongDictType = {
|
||||
dictCStringHash, /* hash function */
|
||||
NULL, /* key dup */
|
||||
NULL, /* val dup */
|
||||
dictCStringCompare, /* key compare */
|
||||
dictKeyDestructor, /* key destructor */
|
||||
nullptr, /* val destructor */
|
||||
nullptr, /* allow to expand */
|
||||
nullptr /* async free destructor */
|
||||
};
|
||||
|
||||
dictType metadataDictType = {
|
||||
dictSdsHash, /* hash function */
|
||||
NULL, /* key dup */
|
||||
NULL, /* val dup */
|
||||
dictSdsKeyCompare, /* key compare */
|
||||
dictSdsDestructor, /* key destructor */
|
||||
dictSdsDestructor, /* val destructor */
|
||||
nullptr, /* allow to expand */
|
||||
nullptr /* async free destructor */
|
||||
};
|
||||
|
||||
|
||||
SnapshotPayloadParseState::ParseStageName SnapshotPayloadParseState::getNextStage() {
|
||||
if (stackParse.empty())
|
||||
return ParseStageName::Global;
|
||||
|
||||
switch (stackParse.back().name)
|
||||
{
|
||||
case ParseStageName::None:
|
||||
return ParseStageName::Global;
|
||||
|
||||
case ParseStageName::Global:
|
||||
if (stackParse.back().arraycur == 0)
|
||||
return ParseStageName::MetaData;
|
||||
else if (stackParse.back().arraycur == 1)
|
||||
return ParseStageName::Databases;
|
||||
break;
|
||||
|
||||
case ParseStageName::MetaData:
|
||||
return ParseStageName::KeyValuePair;
|
||||
|
||||
case ParseStageName::Databases:
|
||||
return ParseStageName::Dataset;
|
||||
|
||||
case ParseStageName::Dataset:
|
||||
return ParseStageName::KeyValuePair;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
throw "Bad protocol: corrupt state";
|
||||
}
|
||||
|
||||
void SnapshotPayloadParseState::flushQueuedKeys() {
|
||||
if (vecqueuedKeys.empty())
|
||||
return;
|
||||
serverAssert(current_database >= 0);
|
||||
|
||||
// TODO: We can't finish parse until all the work functions finish
|
||||
int idb = current_database;
|
||||
serverAssert(vecqueuedKeys.size() == vecqueuedVals.size());
|
||||
auto sizePrev = vecqueuedKeys.size();
|
||||
++insertsInFlight;
|
||||
auto &insertsInFlightTmp = insertsInFlight; // C++ GRRRRRRRRRRRRRRRR, we don't want to capute "this" because that's dangerous
|
||||
if (current_database < cserver.dbnum) {
|
||||
g_pserver->asyncworkqueue->AddWorkFunction([idb, vecqueuedKeys = std::move(this->vecqueuedKeys), vecqueuedKeysCb = std::move(this->vecqueuedKeysCb), vecqueuedVals = std::move(this->vecqueuedVals), vecqueuedValsCb = std::move(this->vecqueuedValsCb), &insertsInFlightTmp, pallocator = m_spallocator.release()]() mutable {
|
||||
g_pserver->db[idb]->bulkStorageInsert(vecqueuedKeys.data(), vecqueuedKeysCb.data(), vecqueuedVals.data(), vecqueuedValsCb.data(), vecqueuedKeys.size());
|
||||
--insertsInFlightTmp;
|
||||
delete pallocator;
|
||||
});
|
||||
} else {
|
||||
// else drop the data
|
||||
vecqueuedKeys.clear();
|
||||
vecqueuedKeysCb.clear();
|
||||
vecqueuedVals.clear();
|
||||
vecqueuedValsCb.clear();
|
||||
// Note: m_spallocator will get free'd when overwritten below
|
||||
}
|
||||
m_spallocator = std::make_unique<SlabAllocator>();
|
||||
cbQueued = 0;
|
||||
vecqueuedKeys.reserve(sizePrev);
|
||||
vecqueuedKeysCb.reserve(sizePrev);
|
||||
vecqueuedVals.reserve(sizePrev);
|
||||
vecqueuedValsCb.reserve(sizePrev);
|
||||
serverAssert(vecqueuedKeys.empty());
|
||||
serverAssert(vecqueuedVals.empty());
|
||||
}
|
||||
|
||||
|
||||
SnapshotPayloadParseState::SnapshotPayloadParseState() {
|
||||
// This is to represent the first array the client is intended to send us
|
||||
ParseStage stage;
|
||||
stage.name = ParseStageName::None;
|
||||
stage.arraylen = 1;
|
||||
stackParse.push_back(stage);
|
||||
|
||||
dictLongLongMetaData = dictCreate(&metadataLongLongDictType, nullptr);
|
||||
dictMetaData = dictCreate(&metadataDictType, nullptr);
|
||||
insertsInFlight = 0;
|
||||
m_spallocator = std::make_unique<SlabAllocator>();
|
||||
}
|
||||
|
||||
SnapshotPayloadParseState::~SnapshotPayloadParseState() {
|
||||
dictRelease(dictLongLongMetaData);
|
||||
dictRelease(dictMetaData);
|
||||
}
|
||||
|
||||
const char *SnapshotPayloadParseState::getStateDebugName(ParseStage stage) {
|
||||
switch (stage.name) {
|
||||
case ParseStageName::None:
|
||||
return "None";
|
||||
|
||||
case ParseStageName::Global:
|
||||
return "Global";
|
||||
|
||||
case ParseStageName::MetaData:
|
||||
return "MetaData";
|
||||
|
||||
case ParseStageName::Databases:
|
||||
return "Databases";
|
||||
|
||||
case ParseStageName::KeyValuePair:
|
||||
return "KeyValuePair";
|
||||
|
||||
case ParseStageName::Dataset:
|
||||
return "Dataset";
|
||||
|
||||
default:
|
||||
return "Unknown";
|
||||
}
|
||||
}
|
||||
|
||||
size_t SnapshotPayloadParseState::depth() const { return stackParse.size(); }
|
||||
|
||||
void SnapshotPayloadParseState::trimState() {
|
||||
while (!stackParse.empty() && (stackParse.back().arraycur == stackParse.back().arraylen))
|
||||
stackParse.pop_back();
|
||||
|
||||
if (stackParse.empty()) {
|
||||
flushQueuedKeys();
|
||||
while (insertsInFlight > 0) {
|
||||
// TODO: ProcessEventsWhileBlocked
|
||||
aeReleaseLock();
|
||||
aeAcquireLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void SnapshotPayloadParseState::pushArray(long long size) {
|
||||
if (stackParse.empty())
|
||||
throw "Bad Protocol: unexpected trailing data";
|
||||
|
||||
if (size == 0) {
|
||||
stackParse.back().arraycur++;
|
||||
return;
|
||||
}
|
||||
|
||||
if (stackParse.back().name == ParseStageName::Databases) {
|
||||
flushQueuedKeys();
|
||||
current_database = stackParse.back().arraycur;
|
||||
}
|
||||
|
||||
ParseStage stage;
|
||||
stage.name = getNextStage();
|
||||
stage.arraylen = size;
|
||||
|
||||
if (stage.name == ParseStageName::Dataset) {
|
||||
g_pserver->db[current_database]->expand(stage.arraylen);
|
||||
}
|
||||
|
||||
// Note: This affects getNextStage so ensure its after
|
||||
stackParse.back().arraycur++;
|
||||
stackParse.push_back(stage);
|
||||
}
|
||||
|
||||
void SnapshotPayloadParseState::pushValue(const char *rgch, long long cch) {
|
||||
if (stackParse.empty())
|
||||
throw "Bad Protocol: unexpected trailing bulk string";
|
||||
|
||||
if (stackParse.back().arraycur >= static_cast<int>(stackParse.back().arrvalues.size()))
|
||||
throw "Bad protocol: Unexpected value";
|
||||
|
||||
auto &stage = stackParse.back();
|
||||
stage.arrvalues[stackParse.back().arraycur].first = (char*)m_spallocator->allocate(cch);
|
||||
stage.arrvalues[stackParse.back().arraycur].second = cch;
|
||||
memcpy(stage.arrvalues[stackParse.back().arraycur].first, rgch, cch);
|
||||
stage.arraycur++;
|
||||
switch (stage.name) {
|
||||
case ParseStageName::KeyValuePair:
|
||||
if (stackParse.size() < 2)
|
||||
throw "Bad Protocol: unexpected bulk string";
|
||||
if (stackParse[stackParse.size()-2].name == ParseStageName::MetaData) {
|
||||
if (stage.arraycur == 2) {
|
||||
// We loaded both pairs
|
||||
if (stage.arrvalues[0].first == nullptr || stage.arrvalues[1].first == nullptr)
|
||||
throw "Bad Protocol: Got array when expecing a string"; // A baddy could make us derefence the vector when its too small
|
||||
|
||||
if (!strcasecmp(stage.arrvalues[0].first, "lua")) {
|
||||
/* Load the script back in memory. */
|
||||
robj *auxval = createStringObject(stage.arrvalues[1].first, stage.arrvalues[1].second);
|
||||
if (luaCreateFunction(NULL,g_pserver->lua,auxval) == NULL) {
|
||||
throw "Can't load Lua script";
|
||||
}
|
||||
} else {
|
||||
dictAdd(dictMetaData, sdsnewlen(stage.arrvalues[0].first, stage.arrvalues[0].second), sdsnewlen(stage.arrvalues[1].first, stage.arrvalues[1].second));
|
||||
}
|
||||
}
|
||||
} else if (stackParse[stackParse.size()-2].name == ParseStageName::Dataset) {
|
||||
if (stage.arraycur == 2) {
|
||||
// We loaded both pairs
|
||||
if (stage.arrvalues[0].first == nullptr || stage.arrvalues[1].first == nullptr)
|
||||
throw "Bad Protocol: Got array when expecing a string"; // A baddy could make us derefence the vector when its too small
|
||||
vecqueuedKeys.push_back(stage.arrvalues[0].first);
|
||||
vecqueuedKeysCb.push_back(stage.arrvalues[0].second);
|
||||
vecqueuedVals.push_back(stage.arrvalues[1].first);
|
||||
vecqueuedValsCb.push_back(stage.arrvalues[1].second);
|
||||
stage.arrvalues[0].first = nullptr;
|
||||
stage.arrvalues[1].first = nullptr;
|
||||
cbQueued += vecqueuedKeysCb.back();
|
||||
cbQueued += vecqueuedValsCb.back();
|
||||
if (cbQueued >= queuedBatchLimit)
|
||||
flushQueuedKeys();
|
||||
}
|
||||
} else {
|
||||
throw "Bad Protocol: unexpected bulk string";
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
throw "Bad Protocol: unexpected bulk string out of KV pair";
|
||||
}
|
||||
}
|
||||
|
||||
void SnapshotPayloadParseState::pushValue(long long value) {
|
||||
if (stackParse.empty())
|
||||
throw "Bad Protocol: unexpected integer value";
|
||||
|
||||
stackParse.back().arraycur++;
|
||||
|
||||
if (stackParse.back().arraycur != 2 || stackParse.back().arrvalues[0].first == nullptr)
|
||||
throw "Bad Protocol: unexpected integer value";
|
||||
|
||||
dictEntry *de = dictAddRaw(dictLongLongMetaData, sdsnewlen(stackParse.back().arrvalues[0].first, stackParse.back().arrvalues[0].second), nullptr);
|
||||
if (de == nullptr)
|
||||
throw "Bad Protocol: metadata field sent twice";
|
||||
de->v.s64 = value;
|
||||
}
|
||||
|
||||
long long SnapshotPayloadParseState::getMetaDataLongLong(const char *szField) const {
|
||||
dictEntry *de = dictFind(dictLongLongMetaData, szField);
|
||||
|
||||
if (de == nullptr) {
|
||||
serverLog(LL_WARNING, "Master did not send field: %s", szField);
|
||||
throw false;
|
||||
}
|
||||
return de->v.s64;
|
||||
}
|
||||
|
||||
sds SnapshotPayloadParseState::getMetaDataStr(const char *szField) const {
|
||||
sdsstring str(szField, strlen(szField));
|
||||
|
||||
dictEntry *de = dictFind(dictMetaData, str.get());
|
||||
if (de == nullptr) {
|
||||
serverLog(LL_WARNING, "Master did not send field: %s", szField);
|
||||
throw false;
|
||||
}
|
||||
return (sds)de->v.val;
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
#pragma once
|
||||
#include <stack>
|
||||
#include <vector>
|
||||
|
||||
class SlabAllocator;
|
||||
|
||||
class SnapshotPayloadParseState {
|
||||
enum class ParseStageName {
|
||||
None,
|
||||
Global,
|
||||
MetaData,
|
||||
Databases,
|
||||
Dataset,
|
||||
KeyValuePair,
|
||||
};
|
||||
struct ParseStage {
|
||||
ParseStageName name;
|
||||
long long arraylen = 0;
|
||||
long long arraycur = 0;
|
||||
std::array<std::pair<char*, size_t>, 2> arrvalues;
|
||||
|
||||
ParseStage() {
|
||||
for (auto &pair : arrvalues) {
|
||||
pair.first = nullptr;
|
||||
pair.second = 0;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<ParseStage> stackParse;
|
||||
|
||||
|
||||
std::vector<char*> vecqueuedKeys;
|
||||
std::vector<size_t> vecqueuedKeysCb;
|
||||
std::vector<char*> vecqueuedVals;
|
||||
std::vector<size_t> vecqueuedValsCb;
|
||||
|
||||
|
||||
std::atomic<int> insertsInFlight;
|
||||
std::unique_ptr<SlabAllocator> m_spallocator;
|
||||
dict *dictLongLongMetaData = nullptr;
|
||||
dict *dictMetaData = nullptr;
|
||||
size_t cbQueued = 0;
|
||||
static const size_t queuedBatchLimit = 64*1024*1024; // 64 MB
|
||||
int current_database = -1;
|
||||
|
||||
ParseStageName getNextStage();
|
||||
void flushQueuedKeys();
|
||||
|
||||
public:
|
||||
SnapshotPayloadParseState();
|
||||
~SnapshotPayloadParseState();
|
||||
|
||||
long long getMetaDataLongLong(const char *field) const;
|
||||
sds getMetaDataStr(const char *szField) const;
|
||||
|
||||
static const char *getStateDebugName(ParseStage stage);
|
||||
|
||||
size_t depth() const;
|
||||
|
||||
void trimState();
|
||||
void pushArray(long long size);
|
||||
void pushValue(const char *rgch, long long cch);
|
||||
void pushValue(long long value);
|
||||
bool shouldThrottle() const { return insertsInFlight > (cserver.cthreads*4); }
|
||||
};
|
||||
@@ -1,216 +0,0 @@
|
||||
#include "server.h"
|
||||
|
||||
uint64_t hashPassthrough(const void *hash) {
|
||||
return static_cast<uint64_t>(reinterpret_cast<uintptr_t>(hash));
|
||||
}
|
||||
|
||||
int hashCompare(void *, const void *key1, const void *key2) {
|
||||
auto diff = (reinterpret_cast<uintptr_t>(key1) - reinterpret_cast<uintptr_t>(key2));
|
||||
return !diff;
|
||||
}
|
||||
|
||||
dictType dbStorageCacheType = {
|
||||
hashPassthrough, /* hash function */
|
||||
NULL, /* key dup */
|
||||
NULL, /* val dup */
|
||||
hashCompare, /* key compare */
|
||||
NULL, /* key destructor */
|
||||
NULL /* val destructor */
|
||||
};
|
||||
|
||||
StorageCache::StorageCache(IStorage *storage, bool fCache)
|
||||
: m_spstorage(storage)
|
||||
{
|
||||
if (fCache)
|
||||
m_pdict = dictCreate(&dbStorageCacheType, nullptr);
|
||||
}
|
||||
|
||||
StorageCache::~StorageCache()
|
||||
{
|
||||
if (m_pdict != nullptr)
|
||||
dictRelease(m_pdict);
|
||||
}
|
||||
|
||||
void StorageCache::clear(void(callback)(void*))
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (m_pdict != nullptr)
|
||||
dictEmpty(m_pdict, callback);
|
||||
m_spstorage->clear();
|
||||
m_collisionCount = 0;
|
||||
}
|
||||
|
||||
void StorageCache::clearAsync()
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (count() == 0)
|
||||
return;
|
||||
if (m_pdict != nullptr) {
|
||||
dict *dSav = m_pdict;
|
||||
m_pdict = dictCreate(&dbStorageCacheType, nullptr);
|
||||
g_pserver->asyncworkqueue->AddWorkFunction([dSav]{
|
||||
dictEmpty(dSav, nullptr);
|
||||
});
|
||||
}
|
||||
m_spstorage->clear();
|
||||
m_collisionCount = 0;
|
||||
}
|
||||
|
||||
void StorageCache::cacheKey(sds key)
|
||||
{
|
||||
if (m_pdict == nullptr)
|
||||
return;
|
||||
uintptr_t hash = dictSdsHash(key);
|
||||
if (dictAdd(m_pdict, reinterpret_cast<void*>(hash), (void*)1) != DICT_OK) {
|
||||
dictEntry *de = dictFind(m_pdict, reinterpret_cast<void*>(hash));
|
||||
serverAssert(de != nullptr);
|
||||
de->v.s64++;
|
||||
m_collisionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
void StorageCache::cacheKey(const char *rgch, size_t cch)
|
||||
{
|
||||
if (m_pdict == nullptr)
|
||||
return;
|
||||
uintptr_t hash = dictGenHashFunction(rgch, (int)cch);
|
||||
if (dictAdd(m_pdict, reinterpret_cast<void*>(hash), (void*)1) != DICT_OK) {
|
||||
dictEntry *de = dictFind(m_pdict, reinterpret_cast<void*>(hash));
|
||||
serverAssert(de != nullptr);
|
||||
de->v.s64++;
|
||||
m_collisionCount++;
|
||||
}
|
||||
}
|
||||
|
||||
bool StorageCache::erase(sds key)
|
||||
{
|
||||
bool result = m_spstorage->erase(key, sdslen(key));
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (result && m_pdict != nullptr)
|
||||
{
|
||||
uint64_t hash = dictSdsHash(key);
|
||||
dictEntry *de = dictFind(m_pdict, reinterpret_cast<void*>(hash));
|
||||
serverAssert(de != nullptr);
|
||||
de->v.s64--;
|
||||
serverAssert(de->v.s64 >= 0);
|
||||
if (de->v.s64 == 0) {
|
||||
dictDelete(m_pdict, reinterpret_cast<void*>(hash));
|
||||
} else {
|
||||
m_collisionCount--;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void StorageCache::insert(sds key, const void *data, size_t cbdata, bool fOverwrite)
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (!fOverwrite && m_pdict != nullptr)
|
||||
{
|
||||
cacheKey(key);
|
||||
}
|
||||
ul.unlock();
|
||||
m_spstorage->insert(key, sdslen(key), (void*)data, cbdata, fOverwrite);
|
||||
}
|
||||
|
||||
long _dictKeyIndex(dict *d, const void *key, uint64_t hash, dictEntry **existing);
|
||||
void StorageCache::bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem)
|
||||
{
|
||||
std::vector<dictEntry*> vechashes;
|
||||
if (m_pdict != nullptr) {
|
||||
vechashes.reserve(celem);
|
||||
|
||||
for (size_t ielem = 0; ielem < celem; ++ielem) {
|
||||
dictEntry *de = (dictEntry*)zmalloc(sizeof(dictEntry));
|
||||
de->key = (void*)dictGenHashFunction(rgkeys[ielem], (int)rgcbkeys[ielem]);
|
||||
de->v.u64 = 1;
|
||||
vechashes.push_back(de);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
bulkInsertsInProgress++;
|
||||
if (m_pdict != nullptr) {
|
||||
for (dictEntry *de : vechashes) {
|
||||
if (dictIsRehashing(m_pdict)) dictRehash(m_pdict,1);
|
||||
/* Get the index of the new element, or -1 if
|
||||
* the element already exists. */
|
||||
long index;
|
||||
if ((index = _dictKeyIndex(m_pdict, de->key, (uint64_t)de->key, nullptr)) == -1) {
|
||||
dictEntry *deLocal = dictFind(m_pdict, de->key);
|
||||
serverAssert(deLocal != nullptr);
|
||||
deLocal->v.s64++;
|
||||
m_collisionCount++;
|
||||
zfree(de);
|
||||
} else {
|
||||
int iht = dictIsRehashing(m_pdict) ? 1 : 0;
|
||||
de->next = m_pdict->ht[iht].table[index];
|
||||
m_pdict->ht[iht].table[index] = de;
|
||||
m_pdict->ht[iht].used++;
|
||||
}
|
||||
}
|
||||
}
|
||||
ul.unlock();
|
||||
|
||||
m_spstorage->bulkInsert(rgkeys, rgcbkeys, rgvals, rgcbvals, celem);
|
||||
|
||||
bulkInsertsInProgress--;
|
||||
}
|
||||
|
||||
const StorageCache *StorageCache::clone()
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
// Clones never clone the cache
|
||||
StorageCache *cacheNew = new StorageCache(const_cast<IStorage*>(m_spstorage->clone()), false /*fCache*/);
|
||||
return cacheNew;
|
||||
}
|
||||
|
||||
void StorageCache::expand(uint64_t slots)
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (m_pdict) {
|
||||
dictExpand(m_pdict, slots);
|
||||
}
|
||||
}
|
||||
|
||||
void StorageCache::retrieve(sds key, IStorage::callbackSingle fn) const
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
if (m_pdict != nullptr)
|
||||
{
|
||||
uint64_t hash = dictSdsHash(key);
|
||||
dictEntry *de = dictFind(m_pdict, reinterpret_cast<void*>(hash));
|
||||
|
||||
if (de == nullptr)
|
||||
return; // Not found
|
||||
}
|
||||
ul.unlock();
|
||||
m_spstorage->retrieve(key, sdslen(key), fn);
|
||||
}
|
||||
|
||||
size_t StorageCache::count() const
|
||||
{
|
||||
std::unique_lock<fastlock> ul(m_lock, std::defer_lock);
|
||||
bool fLocked = ul.try_lock();
|
||||
size_t count = m_spstorage->count();
|
||||
if (m_pdict != nullptr && fLocked) {
|
||||
serverAssert(bulkInsertsInProgress.load(std::memory_order_seq_cst) || count == (dictSize(m_pdict) + m_collisionCount));
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
void StorageCache::beginWriteBatch() {
|
||||
serverAssert(GlobalLocksAcquired()); // Otherwise we deadlock
|
||||
m_spstorage->beginWriteBatch();
|
||||
}
|
||||
|
||||
void StorageCache::emergencyFreeCache() {
|
||||
std::unique_lock<fastlock> ul(m_lock);
|
||||
dict *d = m_pdict;
|
||||
m_pdict = nullptr;
|
||||
if (d != nullptr) {
|
||||
g_pserver->asyncworkqueue->AddWorkFunction([d]{
|
||||
dictRelease(d);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
#pragma once
|
||||
#include "sds.h"
|
||||
|
||||
class StorageCache
|
||||
{
|
||||
std::shared_ptr<IStorage> m_spstorage;
|
||||
dict *m_pdict = nullptr;
|
||||
int m_collisionCount = 0;
|
||||
mutable fastlock m_lock {"StorageCache"};
|
||||
std::atomic<int> bulkInsertsInProgress;
|
||||
|
||||
StorageCache(IStorage *storage, bool fNoCache);
|
||||
|
||||
void cacheKey(sds key);
|
||||
void cacheKey(const char *rgchKey, size_t cchKey);
|
||||
|
||||
struct load_iter_data
|
||||
{
|
||||
StorageCache *cache;
|
||||
IStorageFactory::key_load_iterator itrParent;
|
||||
void *privdataParent;
|
||||
};
|
||||
static void key_load_itr(const char *rgchKey, size_t cchKey, void *privdata)
|
||||
{
|
||||
load_iter_data *data = (load_iter_data*)privdata;
|
||||
data->cache->cacheKey(rgchKey, cchKey);
|
||||
if (data->itrParent)
|
||||
data->itrParent(rgchKey, cchKey, data->privdataParent);
|
||||
}
|
||||
|
||||
public:
|
||||
~StorageCache();
|
||||
|
||||
static StorageCache *create(IStorageFactory *pfactory, int db, IStorageFactory::key_load_iterator fn, void *privdata) {
|
||||
StorageCache *cache = new StorageCache(nullptr, pfactory->FSlow() /*fCache*/);
|
||||
load_iter_data data = {cache, fn, privdata};
|
||||
cache->m_spstorage = std::shared_ptr<IStorage>(pfactory->create(db, key_load_itr, (void*)&data));
|
||||
return cache;
|
||||
}
|
||||
|
||||
void clear(void(callback)(void*));
|
||||
void clearAsync();
|
||||
void insert(sds key, const void *data, size_t cbdata, bool fOverwrite);
|
||||
void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem);
|
||||
void retrieve(sds key, IStorage::callbackSingle fn) const;
|
||||
bool erase(sds key);
|
||||
void emergencyFreeCache();
|
||||
bool keycacheIsEnabled() const { return m_pdict != nullptr; }
|
||||
void expand(uint64_t slots);
|
||||
|
||||
bool enumerate(IStorage::callback fn) const { return m_spstorage->enumerate(fn); }
|
||||
|
||||
void beginWriteBatch();
|
||||
void endWriteBatch() { m_spstorage->endWriteBatch(); }
|
||||
void batch_lock() { return m_spstorage->batch_lock(); }
|
||||
void batch_unlock() { return m_spstorage->batch_unlock(); }
|
||||
|
||||
size_t count() const;
|
||||
|
||||
const StorageCache *clone();
|
||||
};
|
||||
+6
-17
@@ -223,18 +223,12 @@ uint64_t ACLGetCommandCategoryFlagByName(const char *name) {
|
||||
return 0; /* No match. */
|
||||
}
|
||||
|
||||
/* Method for pattern comparison used for the user->patterns list
|
||||
/* Method for passwords/pattern comparison used for the user->passwords list
|
||||
* so that we can search for items with listSearchKey(). */
|
||||
int ACLListMatchSds(void *a, void *b) {
|
||||
return sdscmp((sds)a,(sds)b) == 0;
|
||||
}
|
||||
|
||||
/* Method for password comparison used for the user->passwords list
|
||||
* Like the above, but uses a time independant compare for security reasons */
|
||||
int ACLListMatchSdsSecure(void *a, void* b) {
|
||||
return time_independent_strcmp((sds)a,(sds)b) == 0;
|
||||
}
|
||||
|
||||
/* Method to free list elements from ACL users password/patterns lists. */
|
||||
void ACLListFreeSds(const void *item) {
|
||||
sdsfree((sds)item);
|
||||
@@ -259,7 +253,7 @@ user *ACLCreateUser(const char *name, size_t namelen) {
|
||||
u->passwords = listCreate();
|
||||
u->patterns = listCreate();
|
||||
u->channels = listCreate();
|
||||
listSetMatchMethod(u->passwords,ACLListMatchSdsSecure);
|
||||
listSetMatchMethod(u->passwords,ACLListMatchSds);
|
||||
listSetFreeMethod(u->passwords,ACLListFreeSds);
|
||||
listSetDupMethod(u->passwords,ACLListDupSds);
|
||||
listSetMatchMethod(u->patterns,ACLListMatchSds);
|
||||
@@ -1901,6 +1895,10 @@ void addACLLogEntry(client *c, int reason, int argpos, sds username) {
|
||||
void aclCommand(client *c) {
|
||||
char *sub = szFromObj(c->argv[1]);
|
||||
if (!strcasecmp(sub,"setuser") && c->argc >= 3) {
|
||||
/* Consider information about passwords or permissions
|
||||
* to be sensitive, which will be the arguments for this
|
||||
* subcommand. */
|
||||
preventCommandLogging(c);
|
||||
sds username = szFromObj(c->argv[2]);
|
||||
/* Check username validity. */
|
||||
if (ACLStringHasSpaces(username,sdslen(username))) {
|
||||
@@ -1917,12 +1915,6 @@ void aclCommand(client *c) {
|
||||
user *u = ACLGetUserByName(username,sdslen(username));
|
||||
if (u) ACLCopyUser(tempu, u);
|
||||
|
||||
/* Initially redact all of the arguments to not leak any information
|
||||
* about the user. */
|
||||
for (int j = 2; j < c->argc; j++) {
|
||||
redactClientCommandArgument(c, j);
|
||||
}
|
||||
|
||||
for (int j = 3; j < c->argc; j++) {
|
||||
if (ACLSetUser(tempu,szFromObj(c->argv[j]),sdslen(szFromObj(c->argv[j]))) != C_OK) {
|
||||
const char *errmsg = ACLSetUserStringError();
|
||||
@@ -2256,8 +2248,6 @@ void authCommand(client *c) {
|
||||
addReplyErrorObject(c,shared.syntaxerr);
|
||||
return;
|
||||
}
|
||||
/* Always redact the second argument */
|
||||
redactClientCommandArgument(c, 1);
|
||||
|
||||
/* Handle the two different forms here. The form with two arguments
|
||||
* will just use "default" as username. */
|
||||
@@ -2277,7 +2267,6 @@ void authCommand(client *c) {
|
||||
} else {
|
||||
username = c->argv[1];
|
||||
password = c->argv[2];
|
||||
redactClientCommandArgument(c, 2);
|
||||
}
|
||||
|
||||
if (ACLAuthenticateUser(c,username,password) == C_OK) {
|
||||
|
||||
+13
-86
@@ -53,7 +53,6 @@
|
||||
#include "zmalloc.h"
|
||||
#include "config.h"
|
||||
#include "serverassert.h"
|
||||
#include "readwritelock.h"
|
||||
|
||||
#ifdef USE_MUTEX
|
||||
thread_local int cOwnLock = 0;
|
||||
@@ -88,7 +87,6 @@ mutex_wrapper g_lock;
|
||||
#else
|
||||
fastlock g_lock("AE (global)");
|
||||
#endif
|
||||
readWriteLock g_forkLock("Fork (global)");
|
||||
thread_local aeEventLoop *g_eventLoopThisThread = NULL;
|
||||
|
||||
/* Include the best multiplexing layer supported by this system.
|
||||
@@ -128,13 +126,9 @@ struct aeCommand
|
||||
};
|
||||
void *clientData;
|
||||
};
|
||||
#ifdef PIPE_BUF
|
||||
static_assert(sizeof(aeCommand) <= PIPE_BUF, "aeCommand must be small enough to send atomically through a pipe");
|
||||
#endif
|
||||
|
||||
void aeProcessCmd(aeEventLoop *eventLoop, int fd, void *, int )
|
||||
{
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
|
||||
aeCommand cmd;
|
||||
for (;;)
|
||||
{
|
||||
@@ -156,22 +150,18 @@ void aeProcessCmd(aeEventLoop *eventLoop, int fd, void *, int )
|
||||
|
||||
case AE_ASYNC_OP::PostFunction:
|
||||
{
|
||||
if (cmd.fLock && !ulock.owns_lock()) {
|
||||
g_forkLock.releaseRead();
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
|
||||
if (cmd.fLock)
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
((aePostFunctionProc*)cmd.proc)(cmd.clientData);
|
||||
break;
|
||||
}
|
||||
|
||||
case AE_ASYNC_OP::PostCppFunction:
|
||||
{
|
||||
if (cmd.fLock && !ulock.owns_lock()) {
|
||||
g_forkLock.releaseRead();
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
|
||||
if (cmd.fLock)
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
(*cmd.pfn)();
|
||||
|
||||
delete cmd.pfn;
|
||||
@@ -266,7 +256,7 @@ int aePostFunction(aeEventLoop *eventLoop, std::function<void()> fn, bool fLock,
|
||||
|
||||
aeCommand cmd = {};
|
||||
cmd.op = AE_ASYNC_OP::PostCppFunction;
|
||||
cmd.pfn = new std::function<void()>(fn);
|
||||
cmd.pfn = new (MALLOC_LOCAL) std::function<void()>(fn);
|
||||
cmd.fLock = fLock;
|
||||
|
||||
auto size = write(eventLoop->fdCmdWrite, &cmd, sizeof(cmd));
|
||||
@@ -555,11 +545,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
|
||||
if (te->next)
|
||||
te->next->prev = te->prev;
|
||||
if (te->finalizerProc) {
|
||||
if (!ulock.owns_lock()) {
|
||||
g_forkLock.releaseRead();
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
if (!ulock.owns_lock()) ulock.lock();
|
||||
te->finalizerProc(eventLoop, te->clientData);
|
||||
now = getMonotonicUs();
|
||||
}
|
||||
@@ -579,11 +565,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
|
||||
}
|
||||
|
||||
if (te->when <= now) {
|
||||
if (!ulock.owns_lock()) {
|
||||
g_forkLock.releaseRead();
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
if (!ulock.owns_lock()) ulock.lock();
|
||||
int retval;
|
||||
|
||||
id = te->id;
|
||||
@@ -607,11 +589,8 @@ extern "C" void ProcessEventCore(aeEventLoop *eventLoop, aeFileEvent *fe, int ma
|
||||
{
|
||||
#define LOCK_IF_NECESSARY(fe, tsmask) \
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock); \
|
||||
if (!(fe->mask & tsmask)) { \
|
||||
g_forkLock.releaseRead(); \
|
||||
ulock.lock(); \
|
||||
g_forkLock.acquireRead(); \
|
||||
}
|
||||
if (!(fe->mask & tsmask)) \
|
||||
ulock.lock()
|
||||
|
||||
int fired = 0; /* Number of events fired for current fd. */
|
||||
|
||||
@@ -723,11 +702,8 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
|
||||
|
||||
if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP) {
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
|
||||
if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE)) {
|
||||
g_forkLock.releaseRead();
|
||||
if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE))
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
eventLoop->beforesleep(eventLoop);
|
||||
}
|
||||
|
||||
@@ -738,11 +714,8 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
|
||||
/* After sleep callback. */
|
||||
if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) {
|
||||
std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
|
||||
if (!(eventLoop->aftersleepFlags & AE_SLEEP_THREADSAFE)) {
|
||||
g_forkLock.releaseRead();
|
||||
if (!(eventLoop->aftersleepFlags & AE_SLEEP_THREADSAFE))
|
||||
ulock.lock();
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
eventLoop->aftersleep(eventLoop);
|
||||
}
|
||||
|
||||
@@ -811,27 +784,14 @@ void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep,
|
||||
}
|
||||
|
||||
thread_local spin_worker tl_worker = nullptr;
|
||||
thread_local bool fOwnLockOverride = false;
|
||||
void setAeLockSetThreadSpinWorker(spin_worker worker)
|
||||
{
|
||||
tl_worker = worker;
|
||||
}
|
||||
|
||||
void aeThreadOnline()
|
||||
{
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
|
||||
void aeAcquireLock()
|
||||
{
|
||||
g_forkLock.releaseRead();
|
||||
g_lock.lock(tl_worker);
|
||||
g_forkLock.acquireRead();
|
||||
}
|
||||
|
||||
void aeAcquireForkLock()
|
||||
{
|
||||
g_forkLock.upgradeWrite();
|
||||
}
|
||||
|
||||
int aeTryAcquireLock(int fWeak)
|
||||
@@ -839,47 +799,14 @@ int aeTryAcquireLock(int fWeak)
|
||||
return g_lock.try_lock(!!fWeak);
|
||||
}
|
||||
|
||||
void aeThreadOffline()
|
||||
{
|
||||
g_forkLock.releaseRead();
|
||||
}
|
||||
|
||||
void aeReleaseLock()
|
||||
{
|
||||
g_lock.unlock();
|
||||
}
|
||||
|
||||
void aeSetThreadOwnsLockOverride(int fOverride)
|
||||
{
|
||||
fOwnLockOverride = fOverride;
|
||||
}
|
||||
|
||||
void aeReleaseForkLock()
|
||||
{
|
||||
g_forkLock.downgradeWrite();
|
||||
}
|
||||
|
||||
int aeThreadOwnsLock()
|
||||
{
|
||||
return fOwnLockOverride || g_lock.fOwnLock();
|
||||
}
|
||||
|
||||
int aeLockContested(int threshold)
|
||||
{
|
||||
ticket ticketT;
|
||||
__atomic_load(&g_lock.m_ticket.u, &ticketT.u, __ATOMIC_RELAXED);
|
||||
return ticketT.m_active < static_cast<uint16_t>(ticketT.m_avail - threshold);
|
||||
}
|
||||
|
||||
int aeLockContention()
|
||||
{
|
||||
ticket ticketT;
|
||||
__atomic_load(&g_lock.m_ticket.u, &ticketT.u, __ATOMIC_RELAXED);
|
||||
int32_t avail = ticketT.m_avail;
|
||||
int32_t active = ticketT.m_active;
|
||||
if (avail < active)
|
||||
avail += 0x10000;
|
||||
return avail - active;
|
||||
return g_lock.fOwnLock();
|
||||
}
|
||||
|
||||
void aeClosePipesForForkChild(aeEventLoop *el)
|
||||
@@ -888,4 +815,4 @@ void aeClosePipesForForkChild(aeEventLoop *el)
|
||||
el->fdCmdRead = -1;
|
||||
close(el->fdCmdWrite);
|
||||
el->fdCmdWrite = -1;
|
||||
}
|
||||
}
|
||||
@@ -164,17 +164,10 @@ void aeSetDontWait(aeEventLoop *eventLoop, int noWait);
|
||||
void aeClosePipesForForkChild(aeEventLoop *eventLoop);
|
||||
|
||||
void setAeLockSetThreadSpinWorker(spin_worker worker);
|
||||
void aeThreadOnline();
|
||||
void aeAcquireLock();
|
||||
void aeAcquireForkLock();
|
||||
int aeTryAcquireLock(int fWeak);
|
||||
void aeThreadOffline();
|
||||
void aeReleaseLock();
|
||||
void aeReleaseForkLock();
|
||||
int aeThreadOwnsLock();
|
||||
void aeSetThreadOwnsLockOverride(int fOverride);
|
||||
int aeLockContested(int threshold);
|
||||
int aeLockContention(); // returns the number of instantaneous threads waiting on the lock
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
+8
-56
@@ -36,29 +36,8 @@
|
||||
typedef struct aeApiState {
|
||||
int kqfd;
|
||||
struct kevent *events;
|
||||
|
||||
/* Events mask for merge read and write event.
|
||||
* To reduce memory consumption, we use 2 bits to store the mask
|
||||
* of an event, so that 1 byte will store the mask of 4 events. */
|
||||
char *eventsMask;
|
||||
} aeApiState;
|
||||
|
||||
#define EVENT_MASK_MALLOC_SIZE(sz) (((sz) + 3) / 4)
|
||||
#define EVENT_MASK_OFFSET(fd) ((fd) % 4 * 2)
|
||||
#define EVENT_MASK_ENCODE(fd, mask) (((mask) & 0x3) << EVENT_MASK_OFFSET(fd))
|
||||
|
||||
static inline int getEventMask(const char *eventsMask, int fd) {
|
||||
return (eventsMask[fd/4] >> EVENT_MASK_OFFSET(fd)) & 0x3;
|
||||
}
|
||||
|
||||
static inline void addEventMask(char *eventsMask, int fd, int mask) {
|
||||
eventsMask[fd/4] |= EVENT_MASK_ENCODE(fd, mask);
|
||||
}
|
||||
|
||||
static inline void resetEventMask(char *eventsMask, int fd) {
|
||||
eventsMask[fd/4] &= ~EVENT_MASK_ENCODE(fd, 0x3);
|
||||
}
|
||||
|
||||
static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
aeApiState *state = (aeApiState*)zmalloc(sizeof(aeApiState), MALLOC_LOCAL);
|
||||
|
||||
@@ -75,8 +54,6 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
return -1;
|
||||
}
|
||||
anetCloexec(state->kqfd);
|
||||
state->eventsMask = (char*)zmalloc(EVENT_MASK_MALLOC_SIZE(eventLoop->setsize), MALLOC_LOCAL);
|
||||
memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(eventLoop->setsize));
|
||||
eventLoop->apidata = state;
|
||||
return 0;
|
||||
}
|
||||
@@ -84,9 +61,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
|
||||
static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
|
||||
aeApiState *state = (aeApiState*)eventLoop->apidata;
|
||||
|
||||
state->events = (struct kevent*)zrealloc(state->events, sizeof(struct kevent)*setsize, MALLOC_LOCAL);
|
||||
state->eventsMask = (char*)zrealloc(state->eventsMask, EVENT_MASK_MALLOC_SIZE(setsize), MALLOC_LOCAL);
|
||||
memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(setsize));
|
||||
state->events = (struct kevent*)zrealloc(state->events, sizeof(struct kevent)*setsize, MALLOC_LOCAL);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -95,7 +70,6 @@ static void aeApiFree(aeEventLoop *eventLoop) {
|
||||
|
||||
close(state->kqfd);
|
||||
zfree(state->events);
|
||||
zfree(state->eventsMask);
|
||||
zfree(state);
|
||||
}
|
||||
|
||||
@@ -146,37 +120,15 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
|
||||
if (retval > 0) {
|
||||
int j;
|
||||
|
||||
/* Normally we execute the read event first and then the write event.
|
||||
* When the barrier is set, we will do it reverse.
|
||||
*
|
||||
* However, under kqueue, read and write events would be separate
|
||||
* events, which would make it impossible to control the order of
|
||||
* reads and writes. So we store the event's mask we've got and merge
|
||||
* the same fd events later. */
|
||||
for (j = 0; j < retval; j++) {
|
||||
numevents = retval;
|
||||
for(j = 0; j < numevents; j++) {
|
||||
int mask = 0;
|
||||
struct kevent *e = state->events+j;
|
||||
int fd = e->ident;
|
||||
int mask = 0;
|
||||
|
||||
if (e->filter == EVFILT_READ) mask = AE_READABLE;
|
||||
else if (e->filter == EVFILT_WRITE) mask = AE_WRITABLE;
|
||||
addEventMask(state->eventsMask, fd, mask);
|
||||
}
|
||||
|
||||
/* Re-traversal to merge read and write events, and set the fd's mask to
|
||||
* 0 so that events are not added again when the fd is encountered again. */
|
||||
numevents = 0;
|
||||
for (j = 0; j < retval; j++) {
|
||||
struct kevent *e = state->events+j;
|
||||
int fd = e->ident;
|
||||
int mask = getEventMask(state->eventsMask, fd);
|
||||
|
||||
if (mask) {
|
||||
eventLoop->fired[numevents].fd = fd;
|
||||
eventLoop->fired[numevents].mask = mask;
|
||||
resetEventMask(state->eventsMask, fd);
|
||||
numevents++;
|
||||
}
|
||||
if (e->filter == EVFILT_READ) mask |= AE_READABLE;
|
||||
if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
|
||||
eventLoop->fired[j].fd = e->ident;
|
||||
eventLoop->fired[j].mask = mask;
|
||||
}
|
||||
}
|
||||
return numevents;
|
||||
|
||||
@@ -21,6 +21,7 @@ public:
|
||||
|
||||
if (c != nullptr)
|
||||
{
|
||||
serverAssert(!m_fArmed);
|
||||
serverAssert(c->lock.fOwnLock());
|
||||
|
||||
if (!aeTryAcquireLock(true /*fWeak*/)) // avoid locking the client if we can
|
||||
|
||||
+42
-50
@@ -131,7 +131,7 @@ void installAofRewriteEvent()
|
||||
g_pserver->aof_rewrite_pending = true;
|
||||
int res = aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [] {
|
||||
g_pserver->aof_rewrite_pending = false;
|
||||
if (!g_pserver->aof_stop_sending_diff && g_pserver->aof_pipe_write_data_to_child >= 0)
|
||||
if (g_pserver->aof_pipe_write_data_to_child >= 0)
|
||||
aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL);
|
||||
});
|
||||
if (res != AE_OK)
|
||||
@@ -765,7 +765,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
|
||||
/* In Redis commands are always executed in the context of a client, so in
|
||||
* order to load the append only file we need to create a fake client. */
|
||||
struct client *createAOFClient(void) {
|
||||
struct client *c = new client();
|
||||
struct client *c =(client*) zmalloc(sizeof(*c), MALLOC_LOCAL);
|
||||
|
||||
selectDb(c,0);
|
||||
c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */
|
||||
@@ -778,6 +778,7 @@ struct client *createAOFClient(void) {
|
||||
c->argv = NULL;
|
||||
c->original_argc = 0;
|
||||
c->original_argv = NULL;
|
||||
c->argv_len_sum = 0;
|
||||
c->bufpos = 0;
|
||||
c->fPendingAsyncWrite = FALSE;
|
||||
c->fPendingAsyncWriteHandler = FALSE;
|
||||
@@ -805,7 +806,6 @@ struct client *createAOFClient(void) {
|
||||
c->sockname = NULL;
|
||||
c->resp = 2;
|
||||
c->user = NULL;
|
||||
c->mvccCheckpoint = 0;
|
||||
listSetFreeMethod(c->reply,freeClientReplyValue);
|
||||
listSetDupMethod(c->reply,dupClientReplyValue);
|
||||
fastlock_init(&c->lock, "fake client");
|
||||
@@ -820,7 +820,7 @@ void freeFakeClientArgv(struct client *c) {
|
||||
for (j = 0; j < c->argc; j++)
|
||||
decrRefCount(c->argv[j]);
|
||||
zfree(c->argv);
|
||||
c->argv_len_sumActive = 0;
|
||||
c->argv_len_sum = 0;
|
||||
}
|
||||
|
||||
void freeFakeClient(struct client *c) {
|
||||
@@ -831,7 +831,7 @@ void freeFakeClient(struct client *c) {
|
||||
freeClientOriginalArgv(c);
|
||||
fastlock_unlock(&c->lock);
|
||||
fastlock_free(&c->lock);
|
||||
delete c;
|
||||
zfree(c);
|
||||
}
|
||||
|
||||
/* Replay the append log file. On success C_OK is returned. On non fatal
|
||||
@@ -870,11 +870,6 @@ int loadAppendOnlyFile(char *filename) {
|
||||
fakeClient = createAOFClient();
|
||||
startLoadingFile(fp, filename, RDBFLAGS_AOF_PREAMBLE);
|
||||
|
||||
for (int idb = 0; idb < cserver.dbnum; ++idb)
|
||||
{
|
||||
g_pserver->db[idb]->trackChanges(true);
|
||||
}
|
||||
|
||||
/* Check if this AOF file has an RDB preamble. In that case we need to
|
||||
* load the RDB file and later continue loading the AOF tail. */
|
||||
char sig[5]; /* "REDIS" */
|
||||
@@ -884,7 +879,7 @@ int loadAppendOnlyFile(char *filename) {
|
||||
} else {
|
||||
/* RDB preamble. Pass loading the RDB functions. */
|
||||
rio rdb;
|
||||
rdbSaveInfo rsi;
|
||||
rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
|
||||
|
||||
serverLog(LL_NOTICE,"Reading RDB preamble from AOF file...");
|
||||
if (fseek(fp,0,SEEK_SET) == -1) goto readerr;
|
||||
@@ -1010,11 +1005,6 @@ int loadAppendOnlyFile(char *filename) {
|
||||
}
|
||||
|
||||
loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */
|
||||
for (int idb = 0; idb < cserver.dbnum; ++idb)
|
||||
{
|
||||
if (g_pserver->db[idb]->processChanges(false))
|
||||
g_pserver->db[idb]->commitChanges();
|
||||
}
|
||||
fclose(fp);
|
||||
freeFakeClient(fakeClient);
|
||||
g_pserver->aof_state = old_aof_state;
|
||||
@@ -1548,6 +1538,8 @@ ssize_t aofReadDiffFromParent(void) {
|
||||
}
|
||||
|
||||
int rewriteAppendOnlyFileRio(rio *aof) {
|
||||
dictIterator *di = NULL;
|
||||
dictEntry *de;
|
||||
size_t processed = 0;
|
||||
int j;
|
||||
long key_count = 0;
|
||||
@@ -1555,63 +1547,69 @@ int rewriteAppendOnlyFileRio(rio *aof) {
|
||||
|
||||
for (j = 0; j < cserver.dbnum; j++) {
|
||||
char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
|
||||
redisDb *db = g_pserver->db[j];
|
||||
if (db->size() == 0) continue;
|
||||
redisDb *db = g_pserver->db+j;
|
||||
dict *d = db->dict;
|
||||
if (dictSize(d) == 0) continue;
|
||||
di = dictGetSafeIterator(d);
|
||||
|
||||
/* SELECT the new DB */
|
||||
if (rioWrite(aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr;
|
||||
if (rioWriteBulkLongLong(aof,j) == 0) goto werr;
|
||||
|
||||
/* Iterate this DB writing every entry */
|
||||
bool fComplete = db->iterate([&](const char *keystr, robj *o)->bool{
|
||||
while((de = dictNext(di)) != NULL) {
|
||||
sds keystr;
|
||||
redisObjectStack key;
|
||||
initStaticStringObject(key,(sds)keystr);
|
||||
robj *o = nullptr;
|
||||
|
||||
keystr = (sds)dictGetKey(de);
|
||||
o = (robj*)dictGetVal(de);
|
||||
initStaticStringObject(key,keystr);
|
||||
|
||||
expireEntry *pexpire = getExpire(db,&key);
|
||||
|
||||
/* Save the key and associated value */
|
||||
if (o->type == OBJ_STRING) {
|
||||
/* Emit a SET command */
|
||||
char cmd[]="*3\r\n$3\r\nSET\r\n";
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
|
||||
/* Key and value */
|
||||
if (rioWriteBulkObject(aof,&key) == 0) return false;
|
||||
if (rioWriteBulkObject(aof,o) == 0) return false;
|
||||
if (rioWriteBulkObject(aof,&key) == 0) goto werr;
|
||||
if (rioWriteBulkObject(aof,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_LIST) {
|
||||
if (rewriteListObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteListObject(aof,&key,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_SET) {
|
||||
if (rewriteSetObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteSetObject(aof,&key,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_ZSET) {
|
||||
if (rewriteSortedSetObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteSortedSetObject(aof,&key,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_HASH) {
|
||||
if (rewriteHashObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteHashObject(aof,&key,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_STREAM) {
|
||||
if (rewriteStreamObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteStreamObject(aof,&key,o) == 0) goto werr;
|
||||
} else if (o->type == OBJ_MODULE) {
|
||||
if (rewriteModuleObject(aof,&key,o) == 0) return false;
|
||||
if (rewriteModuleObject(aof,&key,o) == 0) goto werr;
|
||||
} else {
|
||||
serverPanic("Unknown object type");
|
||||
}
|
||||
/* Save the expire time */
|
||||
if (o->FExpires()) {
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
expireEntry *pexpire = db->getExpire(&key);
|
||||
if (pexpire != nullptr) {
|
||||
for (auto &subExpire : *pexpire) {
|
||||
if (subExpire.subkey() == nullptr)
|
||||
{
|
||||
char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n";
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
|
||||
if (rioWriteBulkObject(aof,&key) == 0) return false;
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
|
||||
if (rioWriteBulkObject(aof,&key) == 0) goto werr;
|
||||
}
|
||||
else
|
||||
{
|
||||
char cmd[]="*4\r\n$12\r\nPEXPIREMEMBERAT\r\n";
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
|
||||
if (rioWriteBulkObject(aof,&key) == 0) return false;
|
||||
if (rioWrite(aof,subExpire.subkey(),sdslen(subExpire.subkey())) == 0) return false;
|
||||
if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
|
||||
if (rioWriteBulkObject(aof,&key) == 0) goto werr;
|
||||
if (rioWrite(aof,subExpire.subkey(),sdslen(subExpire.subkey())) == 0) goto werr;
|
||||
}
|
||||
if (rioWriteBulkLongLong(aof,subExpire.when()) == 0) return false; // common
|
||||
if (rioWriteBulkLongLong(aof,subExpire.when()) == 0) goto werr; // common
|
||||
}
|
||||
}
|
||||
|
||||
/* Read some diff from the parent process from time to time. */
|
||||
if (aof->processed_bytes > processed+AOF_READ_DIFF_INTERVAL_BYTES) {
|
||||
processed = aof->processed_bytes;
|
||||
@@ -1628,15 +1626,14 @@ int rewriteAppendOnlyFileRio(rio *aof) {
|
||||
updated_time = now;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
if (!fComplete)
|
||||
goto werr;
|
||||
}
|
||||
dictReleaseIterator(di);
|
||||
di = NULL;
|
||||
}
|
||||
return C_OK;
|
||||
|
||||
werr:
|
||||
if (di) dictReleaseIterator(di);
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
@@ -1675,12 +1672,7 @@ int rewriteAppendOnlyFile(char *filename) {
|
||||
|
||||
if (g_pserver->aof_use_rdb_preamble) {
|
||||
int error;
|
||||
std::vector<const redisDbPersistentDataSnapshot*> vecpdb;
|
||||
for (int idb = 0; idb < cserver.dbnum; ++idb)
|
||||
{
|
||||
vecpdb.push_back(g_pserver->db[idb]);
|
||||
}
|
||||
if (rdbSaveRio(&aof,vecpdb.data(),&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) {
|
||||
if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) {
|
||||
errno = error;
|
||||
goto werr;
|
||||
}
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ const char *ascii_logo =
|
||||
" _ \n"
|
||||
" _-(+)-_ \n"
|
||||
" _-- / \\ --_ \n"
|
||||
" _-- / \\ --_ KeyDB %s (%s/%d) %s bit \n"
|
||||
" _-- / \\ --_ KeyDB %s (%s/%d) %s bit \n"
|
||||
" __-- / \\ --__ \n"
|
||||
" (+) _ / \\ _ (+) Running in %s mode\n"
|
||||
" | -- / \\ -- | Port: %d\n"
|
||||
|
||||
+23
-32
@@ -37,8 +37,8 @@
|
||||
/* Count number of bits set in the binary array pointed by 's' and long
|
||||
* 'count' bytes. The implementation of this function is required to
|
||||
* work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */
|
||||
long long redisPopcount(const void *s, long count) {
|
||||
long long bits = 0;
|
||||
size_t redisPopcount(const void *s, long count) {
|
||||
size_t bits = 0;
|
||||
unsigned char *p = (unsigned char*)s;
|
||||
uint32_t *p4;
|
||||
static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8};
|
||||
@@ -98,11 +98,11 @@ long long redisPopcount(const void *s, long count) {
|
||||
* no zero bit is found, it returns count*8 assuming the string is zero
|
||||
* padded on the right. However if 'bit' is 1 it is possible that there is
|
||||
* not a single set bit in the bitmap. In this special case -1 is returned. */
|
||||
long long redisBitpos(const void *s, unsigned long count, int bit) {
|
||||
long redisBitpos(const void *s, unsigned long count, int bit) {
|
||||
unsigned long *l;
|
||||
unsigned char *c;
|
||||
unsigned long skipval, word = 0, one;
|
||||
long long pos = 0; /* Position of bit, to return to the caller. */
|
||||
long pos = 0; /* Position of bit, to return to the caller. */
|
||||
unsigned long j;
|
||||
int found;
|
||||
|
||||
@@ -405,10 +405,6 @@ void printBits(unsigned char *p, unsigned long count) {
|
||||
#define BITFIELDOP_SET 1
|
||||
#define BITFIELDOP_INCRBY 2
|
||||
|
||||
/* Make sure we don't bit shift too many bits at a time.
|
||||
* Even this amount is probably way too large. */
|
||||
#define BITOP_SHIFT_MAX (1<<30)
|
||||
|
||||
/* This helper function used by GETBIT / SETBIT parses the bit offset argument
|
||||
* making sure an error is returned if it is negative or if it overflows
|
||||
* Redis 512 MB limit for the string value or more (server.proto_max_bulk_len).
|
||||
@@ -416,7 +412,7 @@ void printBits(unsigned char *p, unsigned long count) {
|
||||
* If the 'hash' argument is true, and 'bits is positive, then the command
|
||||
* will also parse bit offsets prefixed by "#". In such a case the offset
|
||||
* is multiplied by 'bits'. This is useful for the BITFIELD command. */
|
||||
int getBitOffsetFromArgument(client *c, robj *o, uint64_t *offset, int hash, int bits) {
|
||||
int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int bits) {
|
||||
long long loffset;
|
||||
const char *err = "bit offset is not an integer or out of range";
|
||||
char *p = szFromObj(o);
|
||||
@@ -441,7 +437,7 @@ int getBitOffsetFromArgument(client *c, robj *o, uint64_t *offset, int hash, int
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
*offset = loffset;
|
||||
*offset = (size_t)loffset;
|
||||
return C_OK;
|
||||
}
|
||||
|
||||
@@ -483,7 +479,7 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
|
||||
* so that the 'maxbit' bit can be addressed. The object is finally
|
||||
* returned. Otherwise if the key holds a wrong type NULL is returned and
|
||||
* an error is sent to the client. */
|
||||
robj *lookupStringForBitCommand(client *c, uint64_t maxbit) {
|
||||
robj *lookupStringForBitCommand(client *c, size_t maxbit) {
|
||||
size_t byte = maxbit >> 3;
|
||||
robj *o = lookupKeyWrite(c->db,c->argv[1]);
|
||||
if (checkType(c,o,OBJ_STRING)) return NULL;
|
||||
@@ -533,7 +529,7 @@ const unsigned char *getObjectReadOnlyString(robj_roptr o, long *len, char *llbu
|
||||
void setbitCommand(client *c) {
|
||||
robj *o;
|
||||
const char *err = "bit is not an integer or out of range";
|
||||
uint64_t bitoffset;
|
||||
size_t bitoffset;
|
||||
ssize_t byte, bit;
|
||||
int byteval, bitval;
|
||||
long on;
|
||||
@@ -572,7 +568,7 @@ void setbitCommand(client *c) {
|
||||
void getbitCommand(client *c) {
|
||||
robj_roptr o;
|
||||
char llbuf[32];
|
||||
uint64_t bitoffset;
|
||||
size_t bitoffset;
|
||||
size_t byte, bit;
|
||||
size_t bitval = 0;
|
||||
|
||||
@@ -646,11 +642,6 @@ void bitopCommand(client *c) {
|
||||
addReplyError(c, "BITOP SHIFT's last parameter must be an integer");
|
||||
return;
|
||||
}
|
||||
if (shift > BITOP_SHIFT_MAX) {
|
||||
addReplyError(c, "BITOP SHIFT value is too large.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (op == BITOP_RSHIFT)
|
||||
shift = -shift;
|
||||
}
|
||||
@@ -691,7 +682,7 @@ void bitopCommand(client *c) {
|
||||
|
||||
if (fShiftOp)
|
||||
{
|
||||
long long newlen = (long long)maxlen + shift/CHAR_BIT;
|
||||
long newlen = (long)maxlen + shift/CHAR_BIT;
|
||||
if (shift > 0 && (shift % CHAR_BIT) != 0)
|
||||
newlen++;
|
||||
|
||||
@@ -703,14 +694,14 @@ void bitopCommand(client *c) {
|
||||
res = (unsigned char*) sdsnewlen(NULL,newlen);
|
||||
if (shift >= 0)
|
||||
{ // left shift
|
||||
long long byteoffset = shift/CHAR_BIT;
|
||||
long byteoffset = shift/CHAR_BIT;
|
||||
memset(res, 0, byteoffset);
|
||||
long long srcLen = newlen - byteoffset - ((shift % CHAR_BIT) ? 1 : 0);
|
||||
long srcLen = newlen - byteoffset - ((shift % CHAR_BIT) ? 1 : 0);
|
||||
|
||||
// now the bitshift+copy
|
||||
unsigned bitshift = shift % CHAR_BIT;
|
||||
unsigned char carry = 0;
|
||||
for (long long iSrc = 0; iSrc < srcLen; ++iSrc)
|
||||
for (long iSrc = 0; iSrc < srcLen; ++iSrc)
|
||||
{
|
||||
res[byteoffset+iSrc] = (src[0][iSrc] << bitshift) | carry;
|
||||
carry = src[0][iSrc] >> (CHAR_BIT - bitshift);
|
||||
@@ -720,14 +711,14 @@ void bitopCommand(client *c) {
|
||||
}
|
||||
else
|
||||
{ // right shift
|
||||
long long byteoffset = -shift/CHAR_BIT;
|
||||
long byteoffset = -shift/CHAR_BIT;
|
||||
unsigned bitshift = -shift % CHAR_BIT;
|
||||
if (bitshift)
|
||||
++byteoffset;
|
||||
res[0] = (src[0][byteoffset] << (CHAR_BIT-bitshift));
|
||||
if (byteoffset > 0)
|
||||
res[0] |= (src[0][byteoffset-1] >> bitshift);
|
||||
for (long long idx = 1; idx < newlen; ++idx)
|
||||
for (long idx = 1; idx < newlen; ++idx)
|
||||
{
|
||||
res[idx] = (src[0][byteoffset+idx] << (CHAR_BIT-bitshift)) | (src[0][byteoffset+idx-1] >> bitshift);
|
||||
}
|
||||
@@ -809,8 +800,8 @@ void bitopCommand(client *c) {
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
/* j is set to the next byte to process by the previous loop. */
|
||||
for (; j < maxlen; j++) {
|
||||
@@ -970,7 +961,7 @@ void bitposCommand(client *c) {
|
||||
addReplyLongLong(c, -1);
|
||||
} else {
|
||||
long bytes = end-start+1;
|
||||
long long pos = redisBitpos(p+start,bytes,bit);
|
||||
long pos = redisBitpos(p+start,bytes,bit);
|
||||
|
||||
/* If we are looking for clear bits, and the user specified an exact
|
||||
* range with start-end, we can't consider the right of the range as
|
||||
@@ -979,11 +970,11 @@ void bitposCommand(client *c) {
|
||||
* So if redisBitpos() returns the first bit outside the range,
|
||||
* we return -1 to the caller, to mean, in the specified range there
|
||||
* is not a single "0" bit. */
|
||||
if (end_given && bit == 0 && pos == (long long)bytes<<3) {
|
||||
if (end_given && bit == 0 && pos == bytes*8) {
|
||||
addReplyLongLong(c,-1);
|
||||
return;
|
||||
}
|
||||
if (pos != -1) pos += (long long)start<<3; /* Adjust for the bytes we skipped. */
|
||||
if (pos != -1) pos += start*8; /* Adjust for the bytes we skipped. */
|
||||
addReplyLongLong(c,pos);
|
||||
}
|
||||
}
|
||||
@@ -1015,12 +1006,12 @@ struct bitfieldOp {
|
||||
* GET subcommand is allowed, other subcommands will return an error. */
|
||||
void bitfieldGeneric(client *c, int flags) {
|
||||
robj_roptr o;
|
||||
uint64_t bitoffset;
|
||||
size_t bitoffset;
|
||||
int j, numops = 0, changes = 0;
|
||||
struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
|
||||
int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
|
||||
int readonly = 1;
|
||||
uint64_t highest_write_offset = 0;
|
||||
size_t highest_write_offset = 0;
|
||||
|
||||
for (j = 2; j < c->argc; j++) {
|
||||
int remargs = c->argc-j-1; /* Remaining args other than current. */
|
||||
@@ -1210,9 +1201,9 @@ void bitfieldGeneric(client *c, int flags) {
|
||||
* object boundaries. */
|
||||
memset(buf,0,9);
|
||||
int i;
|
||||
uint64_t byte = thisop->offset >> 3;
|
||||
size_t byte = thisop->offset >> 3;
|
||||
for (i = 0; i < 9; i++) {
|
||||
if (src == NULL || i+byte >= (uint64_t)strlen) break;
|
||||
if (src == NULL || i+byte >= (size_t)strlen) break;
|
||||
buf[i] = src[i+byte];
|
||||
}
|
||||
|
||||
|
||||
+2
-11
@@ -89,15 +89,8 @@ typedef struct bkinfo {
|
||||
* and will be processed when the client is unblocked. */
|
||||
void blockClient(client *c, int btype) {
|
||||
serverAssert(GlobalLocksAcquired());
|
||||
/* Master client should never be blocked unless pause or module */
|
||||
serverAssert(!(c->flags & CLIENT_MASTER &&
|
||||
btype != BLOCKED_MODULE &&
|
||||
btype != BLOCKED_PAUSE));
|
||||
|
||||
c->flags |= CLIENT_BLOCKED;
|
||||
c->btype = btype;
|
||||
if (btype == BLOCKED_ASYNC)
|
||||
c->casyncOpsPending++;
|
||||
g_pserver->blocked_clients++;
|
||||
g_pserver->blocked_clients_by_type[btype]++;
|
||||
addClientToTimeoutTable(c);
|
||||
@@ -154,7 +147,7 @@ void processUnblockedClients(int iel) {
|
||||
}
|
||||
/* Then process client if it has more data in it's buffer. */
|
||||
if (c->querybuf && sdslen(c->querybuf) > 0) {
|
||||
processInputBuffer(c, true /*fParse*/, CMD_CALL_FULL);
|
||||
processInputBuffer(c, CMD_CALL_FULL);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -201,9 +194,6 @@ void unblockClient(client *c) {
|
||||
} else if (c->btype == BLOCKED_MODULE) {
|
||||
if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c);
|
||||
unblockClientFromModule(c);
|
||||
} else if (c->btype == BLOCKED_ASYNC) {
|
||||
serverAssert(c->casyncOpsPending > 0);
|
||||
c->casyncOpsPending--;
|
||||
} else if (c->btype == BLOCKED_PAUSE) {
|
||||
listDelNode(g_pserver->paused_clients,c->paused_list_node);
|
||||
c->paused_list_node = NULL;
|
||||
@@ -597,6 +587,7 @@ void handleClientsBlockedOnKeys(void) {
|
||||
* lookup, invalidating the first one.
|
||||
* See https://github.com/redis/redis/pull/6554. */
|
||||
serverTL->fixed_time_expire++;
|
||||
updateCachedTime(0);
|
||||
|
||||
/* Serve clients blocked on the key. */
|
||||
robj *o = lookupKeyWrite(rl->db,rl->key);
|
||||
|
||||
+1
-2
@@ -69,7 +69,6 @@ void closeChildInfoPipe(void) {
|
||||
/* Send save data to parent. */
|
||||
void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, const char *pname) {
|
||||
if (g_pserver->child_info_pipe[1] == -1) return;
|
||||
if (g_pserver->rdbThreadVars.fRdbThreadActive && g_pserver->rdbThreadVars.fRdbThreadCancel) return;
|
||||
|
||||
static monotime cow_updated = 0;
|
||||
static uint64_t cow_update_cost = 0;
|
||||
@@ -94,7 +93,7 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress,
|
||||
if (cow) {
|
||||
serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE,
|
||||
"%s: %zu MB of memory used by copy-on-write",
|
||||
pname, cow / (1024 * 1024));
|
||||
pname, data.cow / (1024 * 1024));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-48
@@ -746,7 +746,6 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
|
||||
}
|
||||
connNonBlock(conn);
|
||||
connEnableTcpNoDelay(conn);
|
||||
connKeepAlive(conn,g_pserver->cluster_node_timeout * 2);
|
||||
|
||||
/* Use non-blocking I/O for cluster messages. */
|
||||
serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport);
|
||||
@@ -771,7 +770,7 @@ unsigned long getClusterConnectionsCount(void) {
|
||||
/* We decrement the number of nodes by one, since there is the
|
||||
* "myself" node too in the list. Each node uses two file descriptors,
|
||||
* one incoming and one outgoing, thus the multiplication by 2. */
|
||||
return g_pserver->cluster_enabled && g_pserver->cluster != nullptr ?
|
||||
return g_pserver->cluster_enabled ?
|
||||
((dictSize(g_pserver->cluster->nodes)-1)*2) : 0;
|
||||
}
|
||||
|
||||
@@ -785,7 +784,7 @@ unsigned long getClusterConnectionsCount(void) {
|
||||
* However if the key contains the {...} pattern, only the part between
|
||||
* { and } is hashed. This may be useful in the future to force certain
|
||||
* keys to be in the same node (assuming no resharding is in progress). */
|
||||
unsigned int keyHashSlot(const char *key, int keylen) {
|
||||
unsigned int keyHashSlot(char *key, int keylen) {
|
||||
int s, e; /* start-end indexes of { and } */
|
||||
|
||||
for (s = 0; s < keylen; s++)
|
||||
@@ -1525,9 +1524,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
|
||||
* it's greater than our view but is not in the future
|
||||
* (with 500 milliseconds tolerance) from the POV of our
|
||||
* clock. */
|
||||
mstime_t mstime;
|
||||
__atomic_load(&g_pserver->mstime, &mstime, __ATOMIC_RELAXED);
|
||||
if (pongtime <= (mstime+500) &&
|
||||
if (pongtime <= (g_pserver->mstime+500) &&
|
||||
pongtime > node->pong_received)
|
||||
{
|
||||
node->pong_received = pongtime;
|
||||
@@ -2449,10 +2446,7 @@ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) {
|
||||
if (sdslen(link->sndbuf) == 0 && msglen != 0)
|
||||
{
|
||||
aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [link] {
|
||||
/* The connection could be timed out before this posted function executes (thanks to TCP keepalive).
|
||||
* So check that the connection is still there before setting the write handler, otherwise you segfault */
|
||||
if (link->conn != nullptr)
|
||||
connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1);
|
||||
connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -4129,7 +4123,7 @@ int verifyClusterConfigWithData(void) {
|
||||
|
||||
/* Make sure we only have keys in DB0. */
|
||||
for (j = 1; j < cserver.dbnum; j++) {
|
||||
if (g_pserver->db[j]->size()) return C_ERR;
|
||||
if (dictSize(g_pserver->db[j].dict)) return C_ERR;
|
||||
}
|
||||
|
||||
/* Check that all the slots we see populated memory have a corresponding
|
||||
@@ -4579,7 +4573,7 @@ NULL
|
||||
clusterReplyMultiBulkSlots(c);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"flushslots") && c->argc == 2) {
|
||||
/* CLUSTER FLUSHSLOTS */
|
||||
if (g_pserver->db[0]->size() != 0) {
|
||||
if (dictSize(g_pserver->db[0].dict) != 0) {
|
||||
addReplyError(c,"DB must be empty to perform CLUSTER FLUSHSLOTS.");
|
||||
return;
|
||||
}
|
||||
@@ -4916,7 +4910,7 @@ NULL
|
||||
* slots nor keys to accept to replicate some other node.
|
||||
* Slaves can switch to another master without issues. */
|
||||
if (nodeIsMaster(myself) &&
|
||||
(myself->numslots != 0 || g_pserver->db[0]->size() != 0)) {
|
||||
(myself->numslots != 0 || dictSize(g_pserver->db[0].dict) != 0)) {
|
||||
addReplyError(c,
|
||||
"To set a master the node must be empty and "
|
||||
"without assigned slots.");
|
||||
@@ -5076,7 +5070,7 @@ NULL
|
||||
|
||||
/* Slaves can be reset while containing data, but not master nodes
|
||||
* that must be empty. */
|
||||
if (nodeIsMaster(myself) && c->db->size() != 0) {
|
||||
if (nodeIsMaster(myself) && dictSize(c->db->dict) != 0) {
|
||||
addReplyError(c,"CLUSTER RESET can't be called with "
|
||||
"master nodes containing keys");
|
||||
return;
|
||||
@@ -5092,7 +5086,6 @@ NULL
|
||||
/* -----------------------------------------------------------------------------
|
||||
* DUMP, RESTORE and MIGRATE commands
|
||||
* -------------------------------------------------------------------------- */
|
||||
ssize_t rdbSaveAuxFieldStrStr(rio *rdb, const char *key, const char *val);
|
||||
|
||||
/* Generates a DUMP-format representation of the object 'o', adding it to the
|
||||
* io stream pointed by 'rio'. This function can't fail. */
|
||||
@@ -5105,10 +5098,6 @@ void createDumpPayload(rio *payload, robj_roptr o, robj *key) {
|
||||
rioInitWithBuffer(payload,sdsempty());
|
||||
serverAssert(rdbSaveObjectType(payload,o));
|
||||
serverAssert(rdbSaveObject(payload,o,key));
|
||||
char szT[32];
|
||||
uint64_t mvcc = mvccFromObj(o);
|
||||
snprintf(szT, 32, "%" PRIu64, mvcc);
|
||||
serverAssert(rdbSaveAuxFieldStrStr(payload,"mvcc-tstamp", szT) != -1);
|
||||
|
||||
/* Write the footer, this is how it looks like:
|
||||
* ----------------+---------------------+---------------+
|
||||
@@ -5178,12 +5167,11 @@ void dumpCommand(client *c) {
|
||||
|
||||
/* KEYDB.MVCCRESTORE key mvcc expire serialized-value */
|
||||
void mvccrestoreCommand(client *c) {
|
||||
long long expire;
|
||||
uint64_t mvcc;
|
||||
long long mvcc, expire;
|
||||
robj *key = c->argv[1], *obj = nullptr;
|
||||
int type;
|
||||
|
||||
if (getUnsignedLongLongFromObjectOrReply(c, c->argv[2], &mvcc, "Invalid MVCC Tstamp") != C_OK)
|
||||
if (getLongLongFromObjectOrReply(c, c->argv[2], &mvcc, "Invalid MVCC Tstamp") != C_OK)
|
||||
return;
|
||||
|
||||
if (getLongLongFromObjectOrReply(c, c->argv[3], &expire, "Invalid expire") != C_OK)
|
||||
@@ -5201,7 +5189,7 @@ void mvccrestoreCommand(client *c) {
|
||||
rio payload;
|
||||
rioInitWithBuffer(&payload,szFromObj(c->argv[4]));
|
||||
if (((type = rdbLoadObjectType(&payload)) == -1) ||
|
||||
((obj = rdbLoadObject(type,&payload,szFromObj(key),NULL,OBJ_MVCC_INVALID)) == NULL))
|
||||
((obj = rdbLoadObject(type,&payload,szFromObj(key), OBJ_MVCC_INVALID)) == NULL))
|
||||
{
|
||||
addReplyError(c,"Bad data format");
|
||||
return;
|
||||
@@ -5287,26 +5275,11 @@ void restoreCommand(client *c) {
|
||||
|
||||
rioInitWithBuffer(&payload,szFromObj(c->argv[3]));
|
||||
if (((type = rdbLoadObjectType(&payload)) == -1) ||
|
||||
((obj = rdbLoadObject(type,&payload,szFromObj(key),NULL,OBJ_MVCC_INVALID)) == NULL))
|
||||
((obj = rdbLoadObject(type,&payload,szFromObj(key), OBJ_MVCC_INVALID)) == NULL))
|
||||
{
|
||||
addReplyError(c,"Bad data format");
|
||||
return;
|
||||
}
|
||||
if (rdbLoadType(&payload) == RDB_OPCODE_AUX)
|
||||
{
|
||||
robj *auxkey, *auxval;
|
||||
if ((auxkey = rdbLoadStringObject(&payload)) == NULL) goto eoferr;
|
||||
if ((auxval = rdbLoadStringObject(&payload)) == NULL) {
|
||||
decrRefCount(auxkey);
|
||||
goto eoferr;
|
||||
}
|
||||
if (strcasecmp(szFromObj(auxkey), "mvcc-tstamp") == 0) {
|
||||
setMvccTstamp(obj, strtoull(szFromObj(auxval), nullptr, 10));
|
||||
}
|
||||
decrRefCount(auxkey);
|
||||
decrRefCount(auxval);
|
||||
}
|
||||
eoferr:
|
||||
|
||||
/* Remove the old key if needed. */
|
||||
int deleted = 0;
|
||||
@@ -5486,16 +5459,13 @@ void migrateCommand(client *c) {
|
||||
}
|
||||
j++;
|
||||
password = szFromObj(c->argv[j]);
|
||||
redactClientCommandArgument(c,j);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[j]),"auth2")) {
|
||||
if (moreargs < 2) {
|
||||
addReply(c,shared.syntaxerr);
|
||||
return;
|
||||
}
|
||||
username = szFromObj(c->argv[++j]);
|
||||
redactClientCommandArgument(c,j);
|
||||
password = szFromObj(c->argv[++j]);
|
||||
redactClientCommandArgument(c,j);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[j]),"keys")) {
|
||||
if (sdslen(szFromObj(c->argv[3])) != 0) {
|
||||
addReplyError(c,
|
||||
@@ -5584,13 +5554,12 @@ try_again:
|
||||
/* Create RESTORE payload and generate the protocol to call the command. */
|
||||
for (j = 0; j < num_keys; j++) {
|
||||
long long ttl = 0;
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
expireEntry *pexpire = c->db->getExpire(kv[j]);
|
||||
long long expireat = INVALID_EXPIRE;
|
||||
expireEntry *pexpire = getExpire(c->db,kv[j]);
|
||||
long long expireat = -1;
|
||||
if (pexpire != nullptr)
|
||||
pexpire->FGetPrimaryExpire(&expireat);
|
||||
|
||||
if (expireat != INVALID_EXPIRE) {
|
||||
if (expireat != -1) {
|
||||
ttl = expireat-mstime();
|
||||
if (ttl < 0) {
|
||||
continue;
|
||||
@@ -5876,7 +5845,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
|
||||
multiState *ms, _ms;
|
||||
multiCmd mc;
|
||||
int i, slot = 0, migrating_slot = 0, importing_slot = 0, missing_keys = 0;
|
||||
serverAssert((c->cmd->flags & CMD_ASYNC_OK) || GlobalLocksAcquired());
|
||||
serverAssert(GlobalLocksAcquired());
|
||||
|
||||
/* Allow any key to be set if a module disabled cluster redirections. */
|
||||
if (g_pserver->cluster_module_flags & CLUSTER_MODULE_FLAG_NO_REDIRECTION)
|
||||
@@ -5982,7 +5951,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
|
||||
|
||||
/* Migrating / Importing slot? Count keys we don't have. */
|
||||
if ((migrating_slot || importing_slot) &&
|
||||
lookupKeyRead(g_pserver->db[0],thiskey) == nullptr)
|
||||
lookupKeyRead(&g_pserver->db[0],thiskey) == nullptr)
|
||||
{
|
||||
missing_keys++;
|
||||
}
|
||||
|
||||
+3
-11
@@ -30,16 +30,7 @@ public:
|
||||
zfree(m_data);
|
||||
}
|
||||
|
||||
compactvector(const compactvector &src)
|
||||
{
|
||||
m_celem = src.m_celem;
|
||||
m_max = src.m_max;
|
||||
m_data = (T*)zmalloc(sizeof(T) * m_max, MALLOC_LOCAL);
|
||||
for (size_t ielem = 0; ielem < m_celem; ++ielem)
|
||||
{
|
||||
new (m_data+ielem) T(src[ielem]);
|
||||
}
|
||||
}
|
||||
compactvector(compactvector &) noexcept = delete;
|
||||
|
||||
compactvector(compactvector &&src) noexcept
|
||||
{
|
||||
@@ -51,6 +42,7 @@ public:
|
||||
src.m_max = 0;
|
||||
}
|
||||
|
||||
compactvector &operator=(const compactvector&) noexcept = delete;
|
||||
compactvector &operator=(compactvector &&src) noexcept
|
||||
{
|
||||
zfree(m_data);
|
||||
@@ -69,7 +61,7 @@ public:
|
||||
inline T* end() { return m_data + m_celem; }
|
||||
inline const T* end() const { return m_data + m_celem; }
|
||||
|
||||
T* insert(T* where, const T &val)
|
||||
T* insert(T* where, T &val)
|
||||
{
|
||||
assert(where >= m_data);
|
||||
size_t idx = where - m_data;
|
||||
|
||||
+13
-165
@@ -29,15 +29,10 @@
|
||||
*/
|
||||
|
||||
#include "server.h"
|
||||
#include "storage/rocksdbfactory.h"
|
||||
#include "storage/teststorageprovider.h"
|
||||
#include "cluster.h"
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
#ifdef __linux__
|
||||
#include <sys/sysinfo.h>
|
||||
#endif
|
||||
|
||||
const char *KEYDB_SET_VERSION = KEYDB_REAL_VERSION;
|
||||
|
||||
@@ -105,11 +100,6 @@ configEnum repl_diskless_load_enum[] = {
|
||||
{NULL, 0}
|
||||
};
|
||||
|
||||
configEnum storage_memory_model_enum[] = {
|
||||
{"writeback", STORAGE_WRITEBACK},
|
||||
{"writethrough", STORAGE_WRITETHROUGH},
|
||||
};
|
||||
|
||||
configEnum tls_auth_clients_enum[] = {
|
||||
{"no", TLS_CLIENT_AUTH_NO},
|
||||
{"yes", TLS_CLIENT_AUTH_YES},
|
||||
@@ -343,72 +333,6 @@ void queueLoadModule(sds path, sds *argv, int argc) {
|
||||
listAddNodeTail(g_pserver->loadmodule_queue,loadmod);
|
||||
}
|
||||
|
||||
sds g_sdsProvider = nullptr;
|
||||
sds g_sdsArgs = nullptr;
|
||||
|
||||
bool initializeStorageProvider(const char **err)
|
||||
{
|
||||
try
|
||||
{
|
||||
bool fTest = false;
|
||||
if (g_sdsProvider == nullptr)
|
||||
return true;
|
||||
if (!strcasecmp(g_sdsProvider, "flash") && g_sdsArgs != nullptr)
|
||||
{
|
||||
#ifdef ENABLE_ROCKSDB
|
||||
// Create The Storage Factory (if necessary)
|
||||
serverLog(LL_NOTICE, "Initializing FLASH storage provider (this may take a long time)");
|
||||
adjustOpenFilesLimit();
|
||||
g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0);
|
||||
#else
|
||||
serverLog(LL_WARNING, "To use the flash storage provider please compile KeyDB with ENABLE_FLASH=yes");
|
||||
serverLog(LL_WARNING, "Exiting due to the use of an unsupported storage provider");
|
||||
exit(EXIT_FAILURE);
|
||||
#endif
|
||||
}
|
||||
else if (!strcasecmp(g_sdsProvider, "test") && g_sdsArgs == nullptr)
|
||||
{
|
||||
g_pserver->m_pstorageFactory = new (MALLOC_LOCAL) TestStorageFactory();
|
||||
fTest = true;
|
||||
}
|
||||
|
||||
if (g_pserver->m_pstorageFactory != nullptr && !fTest)
|
||||
{
|
||||
// We need to set max memory to a sane default so keys are actually evicted properly
|
||||
if (g_pserver->maxmemory == 0 && g_pserver->maxmemory_policy == MAXMEMORY_NO_EVICTION)
|
||||
{
|
||||
#ifdef __linux__
|
||||
struct sysinfo sys;
|
||||
if (sysinfo(&sys) == 0)
|
||||
{
|
||||
// By default it's a little under half the memory. This gives sufficient room for background saving
|
||||
g_pserver->maxmemory = static_cast<unsigned long long>(sys.totalram / 2.2);
|
||||
g_pserver->maxmemory_policy = MAXMEMORY_ALLKEYS_LRU;
|
||||
}
|
||||
#else
|
||||
serverLog(LL_WARNING, "Unable to dynamically set maxmemory, please set maxmemory and maxmemory-policy if you are using a storage provier");
|
||||
#endif
|
||||
}
|
||||
else if (g_pserver->maxmemory_policy == MAXMEMORY_NO_EVICTION)
|
||||
{
|
||||
g_pserver->maxmemory_policy = MAXMEMORY_ALLKEYS_LRU;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
*err = "Unknown storage provider";
|
||||
}
|
||||
return g_pserver->m_pstorageFactory != nullptr;
|
||||
}
|
||||
catch(std::string str)
|
||||
{
|
||||
serverLog(LL_WARNING, "ERROR: Failed to initialize %s storage provider. Details to follow below.", g_sdsProvider);
|
||||
serverLog(LL_WARNING, "\t%s", str.c_str());
|
||||
serverLog(LL_WARNING, "KeyDB cannot start. Exiting.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
/* Parse an array of CONFIG_OOM_COUNT sds strings, validate and populate
|
||||
* g_pserver->oom_score_adj_values if valid.
|
||||
*/
|
||||
@@ -435,7 +359,7 @@ static int updateOOMScoreAdjValues(sds *args, const char **err, int apply) {
|
||||
|
||||
if (values[CONFIG_OOM_REPLICA] < values[CONFIG_OOM_MASTER] ||
|
||||
values[CONFIG_OOM_BGCHILD] < values[CONFIG_OOM_REPLICA]) {
|
||||
serverLog(LL_WARNING,
|
||||
serverLog(LOG_WARNING,
|
||||
"The oom-score-adj-values configuration may not work for non-privileged processes! "
|
||||
"Please consult the documentation.");
|
||||
}
|
||||
@@ -747,15 +671,6 @@ void loadServerConfigFromString(char *config) {
|
||||
g_pserver->fActiveReplica = CONFIG_DEFAULT_ACTIVE_REPLICA;
|
||||
err = "argument must be 'yes' or 'no'"; goto loaderr;
|
||||
}
|
||||
} else if (!strcasecmp(argv[0], "tls-allowlist")) {
|
||||
if (argc < 2) {
|
||||
err = "must supply at least one element in the allow list"; goto loaderr;
|
||||
}
|
||||
if (!g_pserver->tls_allowlist.empty()) {
|
||||
err = "tls-allowlist may only be set once"; goto loaderr;
|
||||
}
|
||||
for (int i = 1; i < argc; i++)
|
||||
g_pserver->tls_allowlist.emplace(argv[i], strlen(argv[i]));
|
||||
} else if (!strcasecmp(argv[0], "version-override") && argc == 2) {
|
||||
KEYDB_SET_VERSION = zstrdup(argv[1]);
|
||||
serverLog(LL_WARNING, "Warning version is overriden to: %s\n", KEYDB_SET_VERSION);
|
||||
@@ -763,16 +678,9 @@ void loadServerConfigFromString(char *config) {
|
||||
g_fTestMode = yesnotoi(argv[1]);
|
||||
} else if (!strcasecmp(argv[0],"rdbfuzz-mode")) {
|
||||
// NOP, handled in main
|
||||
} else if (!strcasecmp(argv[0],"storage-provider") && argc >= 2) {
|
||||
g_sdsProvider = sdsdup(argv[1]);
|
||||
if (argc > 2)
|
||||
g_sdsArgs = sdsdup(argv[2]);
|
||||
} else if (!strcasecmp(argv[0],"is-flash-enabled") && argc == 1) {
|
||||
#ifdef ENABLE_ROCKSDB
|
||||
exit(EXIT_SUCCESS);
|
||||
#else
|
||||
exit(EXIT_FAILURE);
|
||||
#endif
|
||||
} else if (!strcasecmp(argv[0],"enable-pro")) {
|
||||
cserver.fUsePro = true;
|
||||
break;
|
||||
} else {
|
||||
err = "Bad directive or wrong number of arguments"; goto loaderr;
|
||||
}
|
||||
@@ -882,18 +790,8 @@ void configSetCommand(client *c) {
|
||||
int err;
|
||||
const char *errstr = NULL;
|
||||
serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2]));
|
||||
|
||||
if (c->argc < 4 || c->argc > 4) {
|
||||
o = nullptr;
|
||||
// Variadic set is only supported for tls-allowlist
|
||||
if (strcasecmp(szFromObj(c->argv[2]), "tls-allowlist")) {
|
||||
addReplySubcommandSyntaxError(c);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
o = c->argv[3];
|
||||
serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3]));
|
||||
}
|
||||
serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3]));
|
||||
o = c->argv[3];
|
||||
|
||||
/* Iterate the configs that are standard */
|
||||
for (standardConfig *config = configs; config->name != NULL; config++) {
|
||||
@@ -902,7 +800,7 @@ void configSetCommand(client *c) {
|
||||
(config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias))))
|
||||
{
|
||||
if (config->flags & SENSITIVE_CONFIG) {
|
||||
redactClientCommandArgument(c,3);
|
||||
preventCommandLogging(c);
|
||||
}
|
||||
if (!config->interface.set(config->data,szFromObj(o),1,&errstr)) {
|
||||
goto badfmt;
|
||||
@@ -1055,12 +953,6 @@ void configSetCommand(client *c) {
|
||||
enableWatchdog(ll);
|
||||
else
|
||||
disableWatchdog();
|
||||
} config_set_special_field("tls-allowlist") {
|
||||
g_pserver->tls_allowlist.clear();
|
||||
for (int i = 3; i < c->argc; ++i) {
|
||||
robj *val = c->argv[i];
|
||||
g_pserver->tls_allowlist.emplace(szFromObj(val), sdslen(szFromObj(val)));
|
||||
}
|
||||
/* Everything else is an error... */
|
||||
} config_set_else {
|
||||
addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
|
||||
@@ -1264,14 +1156,6 @@ void configGetCommand(client *c) {
|
||||
addReplyBulkCString(c, g_pserver->fActiveReplica ? "yes" : "no");
|
||||
matches++;
|
||||
}
|
||||
if (stringmatch(pattern, "tls-allowlist", 1)) {
|
||||
addReplyBulkCString(c,"tls-allowlist");
|
||||
addReplyArrayLen(c, (long)g_pserver->tls_allowlist.size());
|
||||
for (auto &elem : g_pserver->tls_allowlist) {
|
||||
addReplyBulkCBuffer(c, elem.get(), elem.size()); // addReplyBulkSds will free which we absolutely don't want
|
||||
}
|
||||
matches++;
|
||||
}
|
||||
|
||||
setDeferredMapLen(c,replylen,matches);
|
||||
}
|
||||
@@ -1941,20 +1825,6 @@ int rewriteConfig(char *path, int force_all) {
|
||||
rewriteConfigStringOption(state, "version-override",KEYDB_SET_VERSION,KEYDB_REAL_VERSION);
|
||||
rewriteConfigOOMScoreAdjValuesOption(state);
|
||||
|
||||
if (!g_pserver->tls_allowlist.empty()) {
|
||||
sds conf = sdsnew("tls-allowlist ");
|
||||
for (auto &elem : g_pserver->tls_allowlist) {
|
||||
conf = sdscatsds(conf, (sds)elem.get());
|
||||
conf = sdscat(conf, " ");
|
||||
}
|
||||
// trim the trailing space
|
||||
sdsrange(conf, 0, -1);
|
||||
rewriteConfigRewriteLine(state,"tls-allowlist",conf,1 /*force*/);
|
||||
// note: conf is owned by rewriteConfigRewriteLine - no need to free
|
||||
} else {
|
||||
rewriteConfigMarkAsProcessed(state, "tls-allowlist"); // ensure the line is removed if it existed
|
||||
}
|
||||
|
||||
/* Rewrite Sentinel config if in Sentinel mode. */
|
||||
if (g_pserver->sentinel_mode) rewriteConfigSentinelOption(state);
|
||||
|
||||
@@ -2522,12 +2392,8 @@ static int updateJemallocBgThread(int val, int prev, const char **err) {
|
||||
static int updateReplBacklogSize(long long val, long long prev, const char **err) {
|
||||
/* resizeReplicationBacklog sets g_pserver->repl_backlog_size, and relies on
|
||||
* being able to tell when the size changes, so restore prev before calling it. */
|
||||
if (cserver.repl_backlog_disk_size) {
|
||||
*err = "Unable to dynamically resize the backlog because disk backlog is enabled";
|
||||
return 0;
|
||||
}
|
||||
UNUSED(err);
|
||||
g_pserver->repl_backlog_size = prev;
|
||||
g_pserver->repl_backlog_config_size = val;
|
||||
resizeReplicationBacklog(val);
|
||||
return 1;
|
||||
}
|
||||
@@ -2540,7 +2406,7 @@ static int updateMaxmemory(long long val, long long prev, const char **err) {
|
||||
if ((unsigned long long)val < used) {
|
||||
serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET (%llu) is smaller than the current memory usage (%zu). This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy.", g_pserver->maxmemory, used);
|
||||
}
|
||||
performEvictions(false /*fPreSnapshot*/);
|
||||
performEvictions();
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
@@ -2587,7 +2453,7 @@ static int updateMaxclients(long long val, long long prev, const char **err) {
|
||||
adjustOpenFilesLimit();
|
||||
if (g_pserver->maxclients != val) {
|
||||
static char msg[128];
|
||||
snprintf(msg, 128, "The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients);
|
||||
sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients);
|
||||
*err = msg;
|
||||
if (g_pserver->maxclients > prev) {
|
||||
g_pserver->maxclients = prev;
|
||||
@@ -2626,7 +2492,7 @@ static int updateMaxclients(long long val, long long prev, const char **err) {
|
||||
|
||||
if (res != AE_OK){
|
||||
static char msg[128];
|
||||
snprintf(msg, 128,"Failed to post the request to change setsize for Thread %d", iel);
|
||||
sprintf(msg, "Failed to post the request to change setsize for Thread %d", iel);
|
||||
*err = msg;
|
||||
return 0;
|
||||
}
|
||||
@@ -2777,21 +2643,13 @@ standardConfig configs[] = {
|
||||
createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, g_pserver->cluster_enabled, 0, NULL, NULL),
|
||||
createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, g_pserver->aof_enabled, 0, NULL, updateAppendonly),
|
||||
createBoolConfig("cluster-allow-reads-when-down", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_reads_when_down, 0, NULL, NULL),
|
||||
createBoolConfig("delete-on-evict", NULL, MODIFIABLE_CONFIG, cserver.delete_on_evict, 0, NULL, NULL),
|
||||
createBoolConfig("use-fork", NULL, IMMUTABLE_CONFIG, cserver.fForkBgSave, 0, NULL, NULL),
|
||||
createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, fDummy, 0, NULL, NULL),
|
||||
createBoolConfig("time-thread-priority", NULL, IMMUTABLE_CONFIG, cserver.time_thread_priority, 0, NULL, NULL),
|
||||
createBoolConfig("prefetch-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->prefetch_enabled, 1, NULL, NULL),
|
||||
createBoolConfig("allow-rdb-resize-op", NULL, MODIFIABLE_CONFIG, g_pserver->allowRdbResizeOp, 1, NULL, NULL),
|
||||
createBoolConfig("crash-log-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->crashlog_enabled, 1, NULL, updateSighandlerEnabled),
|
||||
createBoolConfig("crash-memcheck-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->memcheck_enabled, 1, NULL, NULL),
|
||||
createBoolConfig("use-exit-on-panic", NULL, MODIFIABLE_CONFIG, g_pserver->use_exit_on_panic, 0, NULL, NULL),
|
||||
createBoolConfig("disable-thp", NULL, MODIFIABLE_CONFIG, g_pserver->disable_thp, 1, NULL, NULL),
|
||||
createBoolConfig("cluster-allow-replica-migration", NULL, MODIFIABLE_CONFIG, g_pserver->cluster_allow_replica_migration, 1, NULL, NULL),
|
||||
createBoolConfig("replica-announced", NULL, MODIFIABLE_CONFIG, g_pserver->replica_announced, 1, NULL, NULL),
|
||||
createBoolConfig("enable-async-commands", NULL, MODIFIABLE_CONFIG, g_pserver->enable_async_commands, 0, NULL, NULL),
|
||||
createBoolConfig("multithread-load-enabled", NULL, MODIFIABLE_CONFIG, g_pserver->multithread_load_enabled, 0, NULL, NULL),
|
||||
createBoolConfig("active-client-balancing", NULL, MODIFIABLE_CONFIG, g_pserver->active_client_balancing, 1, NULL, NULL),
|
||||
|
||||
/* String Configs */
|
||||
createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->acl_filename, "", NULL, NULL),
|
||||
@@ -2807,7 +2665,6 @@ standardConfig configs[] = {
|
||||
createStringConfig("bio_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->bio_cpulist, NULL, NULL, NULL),
|
||||
createStringConfig("aof_rewrite_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->aof_rewrite_cpulist, NULL, NULL, NULL),
|
||||
createStringConfig("bgsave_cpulist", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->bgsave_cpulist, NULL, NULL, NULL),
|
||||
createStringConfig("storage-provider-options", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, cserver.storage_conf, NULL, NULL, NULL),
|
||||
createStringConfig("ignore-warnings", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, g_pserver->ignore_warnings, "", NULL, NULL),
|
||||
createStringConfig("proc-title-template", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, cserver.proc_title_template, CONFIG_DEFAULT_PROC_TITLE_TEMPLATE, isValidProcTitleTemplate, updateProcTitleTemplate),
|
||||
|
||||
@@ -2822,7 +2679,6 @@ standardConfig configs[] = {
|
||||
createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, cserver.verbosity, LL_NOTICE, NULL, NULL),
|
||||
createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, g_pserver->maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL),
|
||||
createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
|
||||
createEnumConfig("storage-cache-mode", NULL, IMMUTABLE_CONFIG, storage_memory_model_enum, cserver.storage_memory_model, STORAGE_WRITETHROUGH, NULL, NULL),
|
||||
createEnumConfig("oom-score-adj", NULL, MODIFIABLE_CONFIG, oom_score_adj_enum, g_pserver->oom_score_adj, OOM_SCORE_ADJ_NO, NULL, updateOOMScoreAdj),
|
||||
createEnumConfig("acl-pubsub-default", NULL, MODIFIABLE_CONFIG, acl_pubsub_default_enum, g_pserver->acl_pubsub_default, USER_FLAG_ALLCHANNELS, NULL, NULL),
|
||||
createEnumConfig("sanitize-dump-payload", NULL, MODIFIABLE_CONFIG, sanitize_dump_payload_enum, cserver.sanitize_dump_payload, SANITIZE_DUMP_NO, NULL, NULL),
|
||||
@@ -2843,7 +2699,7 @@ standardConfig configs[] = {
|
||||
createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->lfu_decay_time, 1, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->slave_priority, 100, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->maxmemory_samples, 16, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("maxmemory-eviction-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->maxmemory_eviction_tenacity, 10, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, cserver.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */
|
||||
createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, g_pserver->slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL),
|
||||
@@ -2861,9 +2717,7 @@ standardConfig configs[] = {
|
||||
createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves),
|
||||
createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, g_pserver->repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves),
|
||||
createIntConfig("min-clients-per-thread", NULL, MODIFIABLE_CONFIG, 0, 400, cserver.thread_min_client_threshold, 20, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("storage-flush-period", NULL, MODIFIABLE_CONFIG, 1, 10000, g_pserver->storage_flush_period, 500, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("replica-quorum", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, g_pserver->repl_quorum, -1, INTEGER_CONFIG, NULL, NULL),
|
||||
createIntConfig("replica-weighting-factor", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, g_pserver->replicaIsolationFactor, 2, INTEGER_CONFIG, NULL, NULL),
|
||||
/* Unsigned int configs */
|
||||
createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, g_pserver->maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients),
|
||||
createUIntConfig("loading-process-events-interval-keys", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_keys, 8192, MEMORY_CONFIG, NULL, NULL),
|
||||
@@ -2881,12 +2735,9 @@ standardConfig configs[] = {
|
||||
createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LLONG_MAX, g_pserver->proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
|
||||
createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
|
||||
createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, g_pserver->repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
|
||||
createLongLongConfig("repl-backlog-disk-reserve", NULL, IMMUTABLE_CONFIG, 0, LLONG_MAX, cserver.repl_backlog_disk_size, 0, MEMORY_CONFIG, NULL, NULL),
|
||||
createLongLongConfig("max-snapshot-slip", NULL, MODIFIABLE_CONFIG, 0, 5000, g_pserver->snapshot_slip, 400, 0, NULL, NULL),
|
||||
|
||||
/* Unsigned Long Long configs */
|
||||
createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
|
||||
createULongLongConfig("maxstorage", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->maxstorage, 0, MEMORY_CONFIG, NULL, NULL),
|
||||
|
||||
/* Size_t configs */
|
||||
createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL),
|
||||
@@ -2908,8 +2759,6 @@ standardConfig configs[] = {
|
||||
createULongConfig("loading-process-events-interval-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, g_pserver->loading_process_events_interval_bytes, 2*1024*1024, MEMORY_CONFIG, NULL, NULL),
|
||||
createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL),
|
||||
createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL),
|
||||
createBoolConfig("force-backlog-disk-reserve", NULL, MODIFIABLE_CONFIG, cserver.force_backlog_disk, 0, NULL, NULL),
|
||||
createBoolConfig("soft-shutdown", NULL, MODIFIABLE_CONFIG, g_pserver->config_soft_shutdown, 0, NULL, NULL),
|
||||
|
||||
#ifdef USE_OPENSSL
|
||||
createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */
|
||||
@@ -2920,7 +2769,6 @@ standardConfig configs[] = {
|
||||
createEnumConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, tls_auth_clients_enum, g_pserver->tls_auth_clients, TLS_CLIENT_AUTH_YES, NULL, NULL),
|
||||
createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool),
|
||||
createBoolConfig("tls-session-caching", NULL, MODIFIABLE_CONFIG, g_pserver->tls_ctx_config.session_caching, 1, NULL, updateTlsCfgBool),
|
||||
createBoolConfig("tls-rotation", NULL, MODIFIABLE_CONFIG, g_pserver->tls_rotation, 0, NULL, updateTlsCfgBool),
|
||||
createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg),
|
||||
createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file, NULL, NULL, updateTlsCfg),
|
||||
createStringConfig("tls-key-file-pass", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, g_pserver->tls_ctx_config.key_file_pass, NULL, NULL, updateTlsCfg),
|
||||
@@ -2964,7 +2812,7 @@ NULL
|
||||
};
|
||||
|
||||
addReplyHelp(c, help);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"set") && c->argc >= 3) {
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"set") && c->argc == 4) {
|
||||
configSetCommand(c);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"get") && c->argc == 3) {
|
||||
configGetCommand(c);
|
||||
|
||||
@@ -36,7 +36,6 @@
|
||||
|
||||
#ifdef __linux__
|
||||
#include <features.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
#define CONFIG_DEFAULT_RDB_FILENAME "dump.rdb"
|
||||
|
||||
+10
-28
@@ -29,7 +29,6 @@
|
||||
|
||||
#include "server.h"
|
||||
#include "connhelpers.h"
|
||||
#include "aelocker.h"
|
||||
|
||||
/* The connections module provides a lean abstraction of network connections
|
||||
* to avoid direct socket and async event management across the Redis code base.
|
||||
@@ -113,7 +112,7 @@ static int connSocketConnect(connection *conn, const char *addr, int port, const
|
||||
conn->state = CONN_STATE_CONNECTING;
|
||||
|
||||
conn->conn_handler = connect_handler;
|
||||
aeCreateFileEvent(serverTL->el, conn->fd, AE_WRITABLE|AE_WRITE_THREADSAFE,
|
||||
aeCreateFileEvent(serverTL->el, conn->fd, AE_WRITABLE,
|
||||
conn->type->ae_handler, conn);
|
||||
|
||||
return C_OK;
|
||||
@@ -226,15 +225,14 @@ static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc fu
|
||||
else
|
||||
conn->flags &= ~CONN_FLAG_WRITE_BARRIER;
|
||||
|
||||
int flags = AE_WRITABLE;
|
||||
if (fThreadSafe)
|
||||
conn->flags |= CONN_FLAG_WRITE_THREADSAFE;
|
||||
else
|
||||
conn->flags &= ~CONN_FLAG_WRITE_THREADSAFE;
|
||||
flags |= AE_WRITE_THREADSAFE;
|
||||
|
||||
if (!conn->write_handler)
|
||||
aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE);
|
||||
else
|
||||
if (aeCreateFileEvent(serverTL->el,conn->fd,AE_WRITABLE|AE_WRITE_THREADSAFE,
|
||||
if (aeCreateFileEvent(serverTL->el,conn->fd,flags,
|
||||
conn->type->ae_handler,conn) == AE_ERR) return C_ERR;
|
||||
return C_OK;
|
||||
}
|
||||
@@ -244,18 +242,17 @@ static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc fu
|
||||
*/
|
||||
static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc func, bool fThreadSafe) {
|
||||
if (func == conn->read_handler) return C_OK;
|
||||
|
||||
|
||||
int flags = AE_READABLE;
|
||||
if (fThreadSafe)
|
||||
conn->flags |= CONN_FLAG_READ_THREADSAFE;
|
||||
else
|
||||
conn->flags &= ~CONN_FLAG_READ_THREADSAFE;
|
||||
flags |= AE_READ_THREADSAFE;
|
||||
|
||||
conn->read_handler = func;
|
||||
if (!conn->read_handler)
|
||||
aeDeleteFileEvent(serverTL->el,conn->fd,AE_READABLE);
|
||||
else
|
||||
if (aeCreateFileEvent(serverTL->el,conn->fd,
|
||||
AE_READABLE|AE_READ_THREADSAFE,conn->type->ae_handler,conn) == AE_ERR) return C_ERR;
|
||||
flags,conn->type->ae_handler,conn) == AE_ERR) return C_ERR;
|
||||
return C_OK;
|
||||
}
|
||||
|
||||
@@ -263,7 +260,7 @@ static const char *connSocketGetLastError(connection *conn) {
|
||||
return strerror(conn->last_errno);
|
||||
}
|
||||
|
||||
void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask)
|
||||
static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask)
|
||||
{
|
||||
UNUSED(el);
|
||||
UNUSED(fd);
|
||||
@@ -282,11 +279,7 @@ void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, in
|
||||
|
||||
if (!conn->write_handler) aeDeleteFileEvent(serverTL->el,conn->fd,AE_WRITABLE);
|
||||
|
||||
{
|
||||
AeLocker locker;
|
||||
locker.arm(nullptr);
|
||||
if (!callHandler(conn, conn->conn_handler)) return;
|
||||
}
|
||||
conn->conn_handler = NULL;
|
||||
}
|
||||
|
||||
@@ -308,27 +301,15 @@ void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, in
|
||||
|
||||
/* Handle normal I/O flows */
|
||||
if (!invert && call_read) {
|
||||
AeLocker lock;
|
||||
if (!(conn->flags & CONN_FLAG_READ_THREADSAFE))
|
||||
lock.arm(nullptr);
|
||||
|
||||
if (!callHandler(conn, conn->read_handler)) return;
|
||||
}
|
||||
/* Fire the writable event. */
|
||||
if (call_write) {
|
||||
AeLocker lock;
|
||||
if (!(conn->flags & CONN_FLAG_WRITE_THREADSAFE))
|
||||
lock.arm(nullptr);
|
||||
|
||||
if (!callHandler(conn, conn->write_handler)) return;
|
||||
}
|
||||
/* If we have to invert the call, fire the readable event now
|
||||
* after the writable one. */
|
||||
if (invert && call_read) {
|
||||
AeLocker lock;
|
||||
if (!(conn->flags & CONN_FLAG_READ_THREADSAFE))
|
||||
lock.arm(nullptr);
|
||||
|
||||
if (!callHandler(conn, conn->read_handler)) return;
|
||||
}
|
||||
}
|
||||
@@ -471,3 +452,4 @@ const char *connGetInfo(connection *conn, char *buf, size_t buf_len) {
|
||||
snprintf(buf, buf_len-1, "fd=%i", conn == NULL ? -1 : conn->fd);
|
||||
return buf;
|
||||
}
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
template <class T>
|
||||
class CowPtr
|
||||
{
|
||||
public:
|
||||
typedef std::shared_ptr<T> RefPtr;
|
||||
|
||||
private:
|
||||
mutable RefPtr m_sp;
|
||||
|
||||
void detach()
|
||||
{
|
||||
if( !( m_sp == nullptr || m_sp.unique() ) ) {
|
||||
m_sp = std::make_shared<T>(*m_sp);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
CowPtr() = default;
|
||||
|
||||
CowPtr(const RefPtr& refptr)
|
||||
: m_sp(refptr)
|
||||
{
|
||||
}
|
||||
|
||||
bool operator==(std::nullptr_t) const
|
||||
{
|
||||
return m_sp == nullptr;
|
||||
}
|
||||
|
||||
bool operator!=(std::nullptr_t) const
|
||||
{
|
||||
return m_sp != nullptr;
|
||||
}
|
||||
|
||||
const T& operator*() const
|
||||
{
|
||||
return *m_sp;
|
||||
}
|
||||
T& operator*()
|
||||
{
|
||||
detach();
|
||||
return *m_sp;
|
||||
}
|
||||
const T* operator->() const
|
||||
{
|
||||
return m_sp.operator->();
|
||||
}
|
||||
T* operator->()
|
||||
{
|
||||
detach();
|
||||
return m_sp.operator->();
|
||||
}
|
||||
};
|
||||
+6
-10
@@ -31,8 +31,7 @@ void cronCommand(client *c)
|
||||
if (getLongLongFromObjectOrReply(c, c->argv[ARG_EXPIRE], &interval, "missing expire time") != C_OK)
|
||||
return;
|
||||
|
||||
long long base;
|
||||
__atomic_load(&g_pserver->mstime, &base, __ATOMIC_ACQUIRE);
|
||||
long long base = g_pserver->mstime;
|
||||
if (getLongLongFromObject(c->argv[ARG_EXPIRE+1], &base) == C_OK) {
|
||||
arg_offset++;
|
||||
std::swap(base, interval);
|
||||
@@ -52,7 +51,6 @@ void cronCommand(client *c)
|
||||
|
||||
if (c->argc < (6 + numkeys)) {
|
||||
addReplyError(c, "Missing arguments or numkeys is too big");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +59,7 @@ void cronCommand(client *c)
|
||||
spjob->interval = (uint64_t)interval;
|
||||
spjob->startTime = (uint64_t)base;
|
||||
spjob->fSingleShot = fSingleShot;
|
||||
spjob->dbNum = dbnumFromDb(c->db);
|
||||
spjob->dbNum = c->db - g_pserver->db;
|
||||
for (long i = 0; i < numkeys; ++i)
|
||||
spjob->veckeys.emplace_back(sdsdup(szFromObj(c->argv[ARG_KEYSTART+i])));
|
||||
for (long i = ARG_KEYSTART + numkeys; i < c->argc; ++i)
|
||||
@@ -117,25 +115,23 @@ void executeCronJobExpireHook(const char *key, robj *o)
|
||||
int dbId = job->dbNum;
|
||||
if (job->fSingleShot)
|
||||
{
|
||||
serverAssert(dbSyncDelete(cFake->db, keyobj));
|
||||
dbSyncDelete(cFake->db, keyobj);
|
||||
}
|
||||
else
|
||||
{
|
||||
job->startTime += job->interval;
|
||||
mstime_t mstime;
|
||||
__atomic_load(&g_pserver->mstime, &mstime, __ATOMIC_ACQUIRE);
|
||||
if (job->startTime < (uint64_t)mstime)
|
||||
if (job->startTime < (uint64_t)g_pserver->mstime)
|
||||
{
|
||||
// If we are more than one interval in the past then fast forward to
|
||||
// the first interval still in the future. If startTime wasn't zero align
|
||||
// this to the original startTime, if it was zero align to now
|
||||
if (job->startTime == job->interval)
|
||||
{ // startTime was 0
|
||||
job->startTime = mstime + job->interval;
|
||||
job->startTime = g_pserver->mstime + job->interval;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto delta = mstime - job->startTime;
|
||||
auto delta = g_pserver->mstime - job->startTime;
|
||||
auto multiple = (delta / job->interval)+1;
|
||||
job->startTime += job->interval * multiple;
|
||||
}
|
||||
|
||||
+281
-1376
File diff suppressed because it is too large.
Load diff
+43
-40
@@ -78,7 +78,6 @@ void bugReportStart(void);
|
||||
void printCrashReport(void);
|
||||
void bugReportEnd(int killViaSignal, int sig);
|
||||
void logStackTrace(void *eip, int uplevel);
|
||||
void getTempFileName(char tmpfile[], int tmpfileNum);
|
||||
|
||||
/* ================================= Debugging ============================== */
|
||||
|
||||
@@ -149,9 +148,8 @@ void mixStringObjectDigest(unsigned char *digest, robj_roptr o) {
|
||||
void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj_roptr o) {
|
||||
uint32_t aux = htonl(o->type);
|
||||
mixDigest(digest,&aux,sizeof(aux));
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
expireEntry *pexpire = db->getExpire(keyobj);
|
||||
long long expiretime = INVALID_EXPIRE;
|
||||
expireEntry *pexpire = getExpire(db,keyobj);
|
||||
long long expiretime = -1;
|
||||
char buf[128];
|
||||
|
||||
if (pexpire != nullptr)
|
||||
@@ -283,7 +281,7 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj
|
||||
serverPanic("Unknown object type");
|
||||
}
|
||||
/* If the key has an expire, add it to the mix */
|
||||
if (expiretime != INVALID_EXPIRE) xorDigest(digest,"!!expire!!",10);
|
||||
if (expiretime != -1) xorDigest(digest,"!!expire!!",10);
|
||||
}
|
||||
|
||||
/* Compute the dataset digest. Since keys, sets elements, hashes elements
|
||||
@@ -293,15 +291,19 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj
|
||||
* as input in order to ensure that a different ordered list will result in
|
||||
* a different digest. */
|
||||
void computeDatasetDigest(unsigned char *final) {
|
||||
unsigned char digest[20];
|
||||
dictIterator *di = NULL;
|
||||
dictEntry *de;
|
||||
int j;
|
||||
uint32_t aux;
|
||||
|
||||
memset(final,0,20); /* Start with a clean result */
|
||||
|
||||
for (j = 0; j < cserver.dbnum; j++) {
|
||||
redisDb *db = g_pserver->db[j];
|
||||
redisDb *db = g_pserver->db+j;
|
||||
|
||||
if (db->size() == 0) continue;
|
||||
if (dictSize(db->dict) == 0) continue;
|
||||
di = dictGetSafeIterator(db->dict);
|
||||
|
||||
/* hash the DB id, so the same dataset moved in a different
|
||||
* DB will lead to a different digest */
|
||||
@@ -309,22 +311,24 @@ void computeDatasetDigest(unsigned char *final) {
|
||||
mixDigest(final,&aux,sizeof(aux));
|
||||
|
||||
/* Iterate this DB writing every entry */
|
||||
db->iterate_threadsafe([final, db](const char *key, robj_roptr o)->bool {
|
||||
unsigned char digest[20];
|
||||
robj *keyobj;
|
||||
while((de = dictNext(di)) != NULL) {
|
||||
sds key;
|
||||
robj *keyobj, *o;
|
||||
|
||||
memset(digest,0,20); /* This key-val digest */
|
||||
key = (sds)dictGetKey(de);
|
||||
keyobj = createStringObject(key,sdslen(key));
|
||||
|
||||
mixDigest(digest,key,sdslen(key));
|
||||
|
||||
o = (robj*)dictGetVal(de);
|
||||
xorObjectDigest(db,keyobj,digest,o);
|
||||
|
||||
/* We can finally xor the key-val digest to the final digest */
|
||||
xorDigest(final,digest,20);
|
||||
decrRefCount(keyobj);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
dictReleaseIterator(di);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,7 +546,7 @@ NULL
|
||||
if (save) {
|
||||
rdbSaveInfo rsi, *rsiptr;
|
||||
rsiptr = rdbPopulateSaveInfo(&rsi);
|
||||
if (rdbSave(nullptr, rsiptr) != C_OK) {
|
||||
if (rdbSaveFile(g_pserver->rdb_filename,rsiptr) != C_OK) {
|
||||
addReply(c,shared.err);
|
||||
return;
|
||||
}
|
||||
@@ -576,14 +580,15 @@ NULL
|
||||
serverLog(LL_WARNING,"Append Only File loaded by DEBUG LOADAOF");
|
||||
addReply(c,shared.ok);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"object") && c->argc == 3) {
|
||||
dictEntry *de;
|
||||
robj *val;
|
||||
const char *strenc;
|
||||
|
||||
val = c->db->find(c->argv[2]);
|
||||
if (val == NULL) {
|
||||
if ((de = dictFind(c->db->dict,ptrFromObj(c->argv[2]))) == NULL) {
|
||||
addReplyErrorObject(c,shared.nokeyerr);
|
||||
return;
|
||||
}
|
||||
val = (robj*)dictGetVal(de);
|
||||
strenc = strEncoding(val->encoding);
|
||||
|
||||
char extra[138] = {0};
|
||||
@@ -627,14 +632,16 @@ NULL
|
||||
strenc, rdbSavedObjectLen(val, c->argv[2]),
|
||||
val->lru, estimateObjectIdleTime(val)/1000, extra);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"sdslen") && c->argc == 3) {
|
||||
auto itr = c->db->find(c->argv[2]);
|
||||
robj *val = itr.val();
|
||||
const char *key = itr.key();
|
||||
dictEntry *de;
|
||||
robj *val;
|
||||
sds key;
|
||||
|
||||
if (val == NULL) {
|
||||
if ((de = dictFind(c->db->dict,ptrFromObj(c->argv[2]))) == NULL) {
|
||||
addReplyErrorObject(c,shared.nokeyerr);
|
||||
return;
|
||||
}
|
||||
val = (robj*)dictGetVal(de);
|
||||
key = (sds)dictGetKey(de);
|
||||
|
||||
if (val->type != OBJ_STRING || !sdsEncodedObject(val)) {
|
||||
addReplyError(c,"Not an sds encoded string.");
|
||||
@@ -644,7 +651,7 @@ NULL
|
||||
"val_sds_len:%lld, val_sds_avail:%lld, val_zmalloc: %lld",
|
||||
(long long) sdslen(key),
|
||||
(long long) sdsavail(key),
|
||||
(long long) sdsZmallocSize((sds)key),
|
||||
(long long) sdsZmallocSize(key),
|
||||
(long long) sdslen(szFromObj(val)),
|
||||
(long long) sdsavail(szFromObj(val)),
|
||||
(long long) getStringObjectSdsUsedMemory(val));
|
||||
@@ -669,8 +676,8 @@ NULL
|
||||
|
||||
if (getPositiveLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK)
|
||||
return;
|
||||
|
||||
c->db->expand(keys);
|
||||
|
||||
dictExpand(c->db->dict,keys);
|
||||
long valsize = 0;
|
||||
if ( c->argc == 5 && getPositiveLongFromObjectOrReply(c, c->argv[4], &valsize, NULL) != C_OK )
|
||||
return;
|
||||
@@ -714,8 +721,8 @@ NULL
|
||||
|
||||
/* We don't use lookupKey because a debug command should
|
||||
* work on logically expired keys */
|
||||
auto itr = c->db->find(c->argv[j]);
|
||||
robj* o = (robj*)(itr == NULL ? NULL : itr.val());
|
||||
dictEntry *de;
|
||||
robj *o = (robj*)((de = dictFind(c->db->dict,ptrFromObj(c->argv[j]))) == NULL) ? NULL : (robj*)dictGetVal(de);
|
||||
if (o) xorObjectDigest(c->db,c->argv[j],digest,o);
|
||||
|
||||
sds d = sdsempty();
|
||||
@@ -734,7 +741,7 @@ NULL
|
||||
} else if (!strcasecmp(name,"double")) {
|
||||
addReplyDouble(c,3.14159265359);
|
||||
} else if (!strcasecmp(name,"bignum")) {
|
||||
addReplyBigNum(c,"1234567999999999999999999999999999999",37);
|
||||
addReplyProto(c,"(1234567999999999999999999999999999999\r\n",40);
|
||||
} else if (!strcasecmp(name,"null")) {
|
||||
addReplyNull(c);
|
||||
} else if (!strcasecmp(name,"array")) {
|
||||
@@ -750,13 +757,11 @@ NULL
|
||||
addReplyBool(c, j == 1);
|
||||
}
|
||||
} else if (!strcasecmp(name,"attrib")) {
|
||||
if (c->resp >= 3) {
|
||||
addReplyAttributeLen(c,1);
|
||||
addReplyBulkCString(c,"key-popularity");
|
||||
addReplyArrayLen(c,2);
|
||||
addReplyBulkCString(c,"key:123");
|
||||
addReplyLongLong(c,90);
|
||||
}
|
||||
addReplyAttributeLen(c,1);
|
||||
addReplyBulkCString(c,"key-popularity");
|
||||
addReplyArrayLen(c,2);
|
||||
addReplyBulkCString(c,"key:123");
|
||||
addReplyLongLong(c,90);
|
||||
/* Attributes are not real replies, so a well formed reply should
|
||||
* also have a normal reply type after the attribute. */
|
||||
addReplyBulkCString(c,"Some real reply following the attribute");
|
||||
@@ -840,11 +845,11 @@ NULL
|
||||
}
|
||||
|
||||
stats = sdscatprintf(stats,"[Dictionary HT]\n");
|
||||
g_pserver->db[dbid]->getStats(buf,sizeof(buf));
|
||||
dictGetStats(buf,sizeof(buf),g_pserver->db[dbid].dict);
|
||||
stats = sdscat(stats,buf);
|
||||
|
||||
stats = sdscatprintf(stats,"[Expires set]\n");
|
||||
g_pserver->db[dbid]->getExpireStats(buf, sizeof(buf));
|
||||
g_pserver->db[dbid].setexpire->getstats(buf, sizeof(buf));
|
||||
stats = sdscat(stats, buf);
|
||||
|
||||
addReplyVerbatim(c,stats,sdslen(stats),"txt");
|
||||
@@ -906,10 +911,6 @@ NULL
|
||||
c->flags &= ~(CLIENT_MASTER | CLIENT_MASTER_FORCE_REPLY);
|
||||
}
|
||||
addReply(c, shared.ok);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]), "get-temp-file") && c->argc == 2) {
|
||||
char tmpfile[256];
|
||||
getTempFileName(tmpfile, g_pserver->rdbThreadVars.tmpfileNum);
|
||||
addReplyBulkCString(c, tmpfile);
|
||||
} else if (!strcasecmp(szFromObj(c->argv[1]),"truncate-repl-backlog") && c->argc == 2) {
|
||||
g_pserver->repl_backlog_idx = 0;
|
||||
g_pserver->repl_backlog_off = g_pserver->master_repl_offset+1;
|
||||
@@ -1790,10 +1791,12 @@ void logCurrentClient(void) {
|
||||
* selected DB, and if so print info about the associated object. */
|
||||
if (cc->argc > 1) {
|
||||
robj *val, *key;
|
||||
dictEntry *de;
|
||||
|
||||
key = getDecodedObject(cc->argv[1]);
|
||||
val = cc->db->find(key);
|
||||
if (val) {
|
||||
de = dictFind(cc->db->dict, ptrFromObj(key));
|
||||
if (de) {
|
||||
val = (robj*)dictGetVal(de);
|
||||
serverLog(LL_WARNING,"key '%s' found in DB containing the following object:", (char*)ptrFromObj(key));
|
||||
serverLogObjectDebugInfo(val);
|
||||
}
|
||||
|
||||
+11
-13
@@ -851,7 +851,6 @@ long defragModule(redisDb *db, dictEntry *kde) {
|
||||
* all the various pointers it has. Returns a stat of how many pointers were
|
||||
* moved. */
|
||||
long defragKey(redisDb *db, dictEntry *de) {
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
sds keysds = (sds)dictGetKey(de);
|
||||
robj *newob, *ob;
|
||||
unsigned char *newzl;
|
||||
@@ -865,14 +864,15 @@ long defragKey(redisDb *db, dictEntry *de) {
|
||||
if (newsds)
|
||||
{
|
||||
defragged++, de->key = newsds;
|
||||
if (!db->setexpire()->empty()) {
|
||||
bool fReplaced = replaceSatelliteOSetKeyPtr(*const_cast<expireset*>(db->setexpire()), keysds, newsds);
|
||||
if (!db->setexpire->empty()) {
|
||||
bool fReplaced = replaceSatelliteOSetKeyPtr(*db->setexpire, keysds, newsds);
|
||||
serverAssert(fReplaced == ob->FExpires());
|
||||
} else {
|
||||
serverAssert(!ob->FExpires());
|
||||
}
|
||||
}
|
||||
|
||||
/* Try to defrag robj and / or string value. */
|
||||
if ((newob = activeDefragStringOb(ob, &defragged))) {
|
||||
de->v.val = newob;
|
||||
ob = newob;
|
||||
@@ -989,8 +989,9 @@ long defragOtherGlobals() {
|
||||
|
||||
/* returns 0 more work may or may not be needed (see non-zero cursor),
|
||||
* and 1 if time is up and more work is needed. */
|
||||
int defragLaterItem(sds key, robj *ob, unsigned long *cursor, long long endtime) {
|
||||
if (ob) {
|
||||
int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) {
|
||||
if (de) {
|
||||
robj *ob = (robj*)dictGetVal(de);
|
||||
if (ob->type == OBJ_LIST) {
|
||||
return scanLaterList(ob, cursor, endtime, &g_pserver->stat_active_defrag_hits);
|
||||
} else if (ob->type == OBJ_SET) {
|
||||
@@ -1002,9 +1003,7 @@ int defragLaterItem(sds key, robj *ob, unsigned long *cursor, long long endtime)
|
||||
} else if (ob->type == OBJ_STREAM) {
|
||||
return scanLaterStreamListpacks(ob, cursor, endtime, &g_pserver->stat_active_defrag_hits);
|
||||
} else if (ob->type == OBJ_MODULE) {
|
||||
redisObjectStack oT;
|
||||
initStaticStringObject(oT, key);
|
||||
return moduleLateDefrag(&oT, ob, cursor, endtime, &g_pserver->stat_active_defrag_hits);
|
||||
return moduleLateDefrag((robj*)dictGetKey(de), ob, cursor, endtime, &g_pserver->stat_active_defrag_hits);
|
||||
} else {
|
||||
*cursor = 0; /* object type may have changed since we schedule it for later */
|
||||
}
|
||||
@@ -1049,11 +1048,11 @@ int defragLaterStep(redisDb *db, long long endtime) {
|
||||
}
|
||||
|
||||
/* each time we enter this function we need to fetch the key from the dict again (if it still exists) */
|
||||
robj *o = db->find(defrag_later_current_key);
|
||||
dictEntry *de = dictFind(db->dict, defrag_later_current_key);
|
||||
key_defragged = g_pserver->stat_active_defrag_hits;
|
||||
do {
|
||||
int quit = 0;
|
||||
if (defragLaterItem(defrag_later_current_key, o, &defrag_later_cursor, endtime))
|
||||
if (defragLaterItem(de, &defrag_later_cursor, endtime))
|
||||
quit = 1; /* time is up, we didn't finish all the work */
|
||||
|
||||
/* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields
|
||||
@@ -1201,7 +1200,7 @@ void activeDefragCycle(void) {
|
||||
start_stat = g_pserver->stat_active_defrag_hits;
|
||||
}
|
||||
|
||||
db = g_pserver->db[current_db];
|
||||
db = &g_pserver->db[current_db];
|
||||
cursor = 0;
|
||||
}
|
||||
|
||||
@@ -1212,8 +1211,7 @@ void activeDefragCycle(void) {
|
||||
break; /* this will exit the function and we'll continue on the next cycle */
|
||||
}
|
||||
|
||||
// we actually look at the objects too but defragScanCallback can handle missing values
|
||||
cursor = dictScan(db->dictUnsafeKeyOnly(), cursor, defragScanCallback, defragDictBucketCallback, db);
|
||||
cursor = dictScan(db->dict, cursor, defragScanCallback, defragDictBucketCallback, db);
|
||||
|
||||
/* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys
|
||||
* (if we have a lot of pointers in one hash bucket or rehasing),
|
||||
|
||||
+48
-426
@@ -42,7 +42,6 @@
|
||||
#include <stdarg.h>
|
||||
#include <limits.h>
|
||||
#include <sys/time.h>
|
||||
#include <algorithm>
|
||||
|
||||
#include "dict.h"
|
||||
#include "zmalloc.h"
|
||||
@@ -63,15 +62,13 @@ static unsigned int dict_force_resize_ratio = 5;
|
||||
|
||||
static int _dictExpandIfNeeded(dict *ht);
|
||||
static unsigned long _dictNextPower(unsigned long size);
|
||||
long _dictKeyIndex(dict *ht, const void *key, uint64_t hash, dictEntry **existing);
|
||||
static long _dictKeyIndex(dict *ht, const void *key, uint64_t hash, dictEntry **existing);
|
||||
static int _dictInit(dict *ht, dictType *type, void *privDataPtr);
|
||||
|
||||
/* -------------------------- hash functions -------------------------------- */
|
||||
|
||||
static uint8_t dict_hash_function_seed[16];
|
||||
|
||||
extern "C" void asyncFreeDictTable(dictEntry **de);
|
||||
|
||||
void dictSetHashFunctionSeed(uint8_t *seed) {
|
||||
memcpy(dict_hash_function_seed,seed,sizeof(dict_hash_function_seed));
|
||||
}
|
||||
@@ -126,31 +123,37 @@ int _dictInit(dict *d, dictType *type,
|
||||
d->privdata = privDataPtr;
|
||||
d->rehashidx = -1;
|
||||
d->pauserehash = 0;
|
||||
d->asyncdata = nullptr;
|
||||
d->refcount = 1;
|
||||
d->noshrink = false;
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
/* Resize the table to the minimal size that contains all the elements,
|
||||
* but with the invariant of a USED/BUCKETS ratio near to <= 1 */
|
||||
int dictResize(dict *d)
|
||||
{
|
||||
unsigned long minimal;
|
||||
|
||||
if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
|
||||
minimal = d->ht[0].used;
|
||||
if (minimal < DICT_HT_INITIAL_SIZE)
|
||||
minimal = DICT_HT_INITIAL_SIZE;
|
||||
return dictExpand(d, minimal);
|
||||
}
|
||||
|
||||
/* Expand or create the hash table,
|
||||
* when malloc_failed is non-NULL, it'll avoid panic if malloc fails (in which case it'll be set to 1).
|
||||
* Returns DICT_OK if expand was performed, and DICT_ERR if skipped. */
|
||||
int _dictExpand(dict *d, unsigned long size, bool fShrink, int* malloc_failed)
|
||||
int _dictExpand(dict *d, unsigned long size, int* malloc_failed)
|
||||
{
|
||||
if (malloc_failed) *malloc_failed = 0;
|
||||
|
||||
/* the size is invalid if it is smaller than the number of
|
||||
* elements already inside the hash table */
|
||||
if (dictIsRehashing(d) || (d->ht[0].used > size && !fShrink) || size == 0)
|
||||
if (dictIsRehashing(d) || d->ht[0].used > size)
|
||||
return DICT_ERR;
|
||||
|
||||
dictht n; /* the new hash table */
|
||||
unsigned long realsize = _dictNextPower(size);
|
||||
|
||||
/* Detect overflows */
|
||||
if (realsize < size || realsize * sizeof(dictEntry*) < realsize)
|
||||
return DICT_ERR;
|
||||
|
||||
/* Rehashing to the same table size is not useful. */
|
||||
if (realsize == d->ht[0].size) return DICT_ERR;
|
||||
|
||||
@@ -181,169 +184,15 @@ int _dictExpand(dict *d, unsigned long size, bool fShrink, int* malloc_failed)
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
/* Resize the table to the minimal size that contains all the elements,
|
||||
* but with the invariant of a USED/BUCKETS ratio near to <= 1 */
|
||||
int dictResize(dict *d)
|
||||
{
|
||||
unsigned long minimal;
|
||||
|
||||
if (!dict_can_resize || dictIsRehashing(d)) return DICT_ERR;
|
||||
minimal = d->ht[0].used;
|
||||
if (minimal < DICT_HT_INITIAL_SIZE)
|
||||
minimal = DICT_HT_INITIAL_SIZE;
|
||||
return _dictExpand(d, minimal, false /*fShirnk*/, nullptr);
|
||||
}
|
||||
|
||||
int dictMerge(dict *dst, dict *src)
|
||||
{
|
||||
#define MERGE_BLOCK_SIZE 4
|
||||
dictEntry *rgdeT[MERGE_BLOCK_SIZE];
|
||||
|
||||
assert(dst != src);
|
||||
if (dictSize(src) == 0)
|
||||
return DICT_OK;
|
||||
|
||||
if (dictSize(dst) == 0)
|
||||
{
|
||||
dict::swap(*dst, *src);
|
||||
std::swap(dst->pauserehash, src->pauserehash);
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
size_t expectedSize = dictSize(src) + dictSize(dst);
|
||||
if (dictSize(src) > dictSize(dst) && src->asyncdata == nullptr && dst->asyncdata == nullptr)
|
||||
{
|
||||
dict::swap(*dst, *src);
|
||||
std::swap(dst->pauserehash, src->pauserehash);
|
||||
}
|
||||
|
||||
if (!dictIsRehashing(dst) && !dictIsRehashing(src))
|
||||
{
|
||||
if (dst->ht[0].size >= src->ht[0].size)
|
||||
{
|
||||
dst->ht[1] = dst->ht[0];
|
||||
dst->ht[0] = src->ht[0];
|
||||
}
|
||||
else
|
||||
{
|
||||
dst->ht[1] = src->ht[0];
|
||||
}
|
||||
_dictReset(&src->ht[0]);
|
||||
dst->rehashidx = 0;
|
||||
assert(dictIsRehashing(dst));
|
||||
assert((dictSize(src)+dictSize(dst)) == expectedSize);
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
if (!dictIsRehashing(src) && dictSize(src) > 0 &&
|
||||
(src->ht[0].size == dst->ht[0].size || src->ht[0].size == dst->ht[1].size))
|
||||
{
|
||||
auto &htDst = (src->ht[0].size == dst->ht[0].size) ? dst->ht[0] : dst->ht[1];
|
||||
|
||||
assert(src->ht[0].size == htDst.size);
|
||||
for (size_t ide = 0; ide < src->ht[0].size; ide += MERGE_BLOCK_SIZE)
|
||||
{
|
||||
if (src->ht[0].used == 0)
|
||||
break;
|
||||
|
||||
for (int dde = 0; dde < MERGE_BLOCK_SIZE; ++dde) {
|
||||
rgdeT[dde] = src->ht[0].table[ide + dde];
|
||||
src->ht[0].table[ide + dde] = nullptr;
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
bool fAnyFound = false;
|
||||
for (int dde = 0; dde < MERGE_BLOCK_SIZE; ++dde) {
|
||||
if (rgdeT[dde] == nullptr)
|
||||
continue;
|
||||
dictEntry *deNext = rgdeT[dde]->next;
|
||||
rgdeT[dde]->next = htDst.table[ide+dde];
|
||||
htDst.table[ide+dde] = rgdeT[dde];
|
||||
rgdeT[dde] = deNext;
|
||||
htDst.used++;
|
||||
src->ht[0].used--;
|
||||
|
||||
fAnyFound = fAnyFound || (deNext != nullptr);
|
||||
}
|
||||
|
||||
if (!fAnyFound)
|
||||
break;
|
||||
}
|
||||
}
|
||||
// If we copied to the base hash table of a rehashing dst, reset the rehash
|
||||
if (dictIsRehashing(dst) && src->ht[0].size == dst->ht[0].size)
|
||||
dst->rehashidx = 0;
|
||||
assert(dictSize(src) == 0);
|
||||
assert((dictSize(src)+dictSize(dst)) == expectedSize);
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
_dictExpand(dst, dictSize(dst)+dictSize(src), false /* fShrink */, nullptr); // start dst rehashing if necessary
|
||||
auto &htDst = dictIsRehashing(dst) ? dst->ht[1] : dst->ht[0];
|
||||
for (int iht = 0; iht < 2; ++iht)
|
||||
{
|
||||
for (size_t ide = 0; ide < src->ht[iht].size; ide += MERGE_BLOCK_SIZE)
|
||||
{
|
||||
if (src->ht[iht].used == 0)
|
||||
break;
|
||||
|
||||
for (int dde = 0; dde < MERGE_BLOCK_SIZE; ++dde) {
|
||||
rgdeT[dde] = src->ht[iht].table[ide + dde];
|
||||
src->ht[iht].table[ide + dde] = nullptr;
|
||||
}
|
||||
|
||||
for (;;) {
|
||||
bool fAnyFound = false;
|
||||
for (int dde = 0; dde < MERGE_BLOCK_SIZE; ++dde) {
|
||||
if (rgdeT[dde] == nullptr)
|
||||
continue;
|
||||
uint64_t h = dictHashKey(dst, rgdeT[dde]->key) & htDst.sizemask;
|
||||
dictEntry *deNext = rgdeT[dde]->next;
|
||||
rgdeT[dde]->next = htDst.table[h];
|
||||
htDst.table[h] = rgdeT[dde];
|
||||
rgdeT[dde] = deNext;
|
||||
htDst.used++;
|
||||
src->ht[iht].used--;
|
||||
fAnyFound = fAnyFound || (deNext != nullptr);
|
||||
}
|
||||
if (!fAnyFound)
|
||||
break;
|
||||
}
|
||||
#if 0
|
||||
dictEntry *de = src->ht[iht].table[ide];
|
||||
src->ht[iht].table[ide] = nullptr;
|
||||
while (de != nullptr)
|
||||
{
|
||||
dictEntry *deNext = de->next;
|
||||
|
||||
uint64_t h = dictHashKey(dst, de->key) & htDst.sizemask;
|
||||
de->next = htDst.table[h];
|
||||
htDst.table[h] = de;
|
||||
htDst.used++;
|
||||
|
||||
de = deNext;
|
||||
src->ht[iht].used--;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
assert((dictSize(src)+dictSize(dst)) == expectedSize);
|
||||
return DICT_OK;
|
||||
}
|
||||
|
||||
/* return DICT_ERR if expand was not performed */
|
||||
int dictExpand(dict *d, unsigned long size, bool fShrink) {
|
||||
// External expand likely means mass insertion, and we don't want to shrink during that
|
||||
d->noshrink = true;
|
||||
return _dictExpand(d, size, fShrink, NULL);
|
||||
int dictExpand(dict *d, unsigned long size) {
|
||||
return _dictExpand(d, size, NULL);
|
||||
}
|
||||
|
||||
/* return DICT_ERR if expand failed due to memory allocation failure */
|
||||
int dictTryExpand(dict *d, unsigned long size, bool fShrink) {
|
||||
int dictTryExpand(dict *d, unsigned long size) {
|
||||
int malloc_failed;
|
||||
// External expand likely means mass insertion, and we don't want to shrink during that
|
||||
d->noshrink = true;
|
||||
_dictExpand(d, size, fShrink, &malloc_failed);
|
||||
_dictExpand(d, size, &malloc_failed);
|
||||
return malloc_failed? DICT_ERR : DICT_OK;
|
||||
}
|
||||
|
||||
@@ -359,7 +208,6 @@ int dictTryExpand(dict *d, unsigned long size, bool fShrink) {
|
||||
int dictRehash(dict *d, int n) {
|
||||
int empty_visits = n*10; /* Max number of empty buckets to visit. */
|
||||
if (!dictIsRehashing(d)) return 0;
|
||||
if (d->asyncdata) return 0;
|
||||
|
||||
while(n-- && d->ht[0].used != 0) {
|
||||
dictEntry *de, *nextde;
|
||||
@@ -390,7 +238,7 @@ int dictRehash(dict *d, int n) {
|
||||
|
||||
/* Check if we already rehashed the whole table... */
|
||||
if (d->ht[0].used == 0) {
|
||||
asyncFreeDictTable(d->ht[0].table);
|
||||
zfree(d->ht[0].table);
|
||||
d->ht[0] = d->ht[1];
|
||||
_dictReset(&d->ht[1]);
|
||||
d->rehashidx = -1;
|
||||
@@ -401,158 +249,6 @@ int dictRehash(dict *d, int n) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
dictAsyncRehashCtl::dictAsyncRehashCtl(struct dict *d, dictAsyncRehashCtl *next) : dict(d), next(next) {
|
||||
queue.reserve(c_targetQueueSize);
|
||||
__atomic_fetch_add(&d->refcount, 1, __ATOMIC_ACQ_REL);
|
||||
this->rehashIdxBase = d->rehashidx;
|
||||
}
|
||||
|
||||
dictAsyncRehashCtl *dictRehashAsyncStart(dict *d, int buckets) {
|
||||
assert(d->type->asyncfree != nullptr);
|
||||
if (!dictIsRehashing(d) || d->pauserehash != 0) return nullptr;
|
||||
|
||||
d->asyncdata = new dictAsyncRehashCtl(d, d->asyncdata);
|
||||
|
||||
int empty_visits = buckets*10;
|
||||
|
||||
while (d->asyncdata->queue.size() < (size_t)buckets && (size_t)d->rehashidx < d->ht[0].size) {
|
||||
dictEntry *de;
|
||||
|
||||
/* Note that rehashidx can't overflow as we are sure there are more
|
||||
* elements because ht[0].used != 0 */
|
||||
while(d->ht[0].table[d->rehashidx] == NULL) {
|
||||
d->rehashidx++;
|
||||
if (--empty_visits == 0) goto LDone;
|
||||
if ((size_t)d->rehashidx >= d->ht[0].size) goto LDone;
|
||||
}
|
||||
|
||||
de = d->ht[0].table[d->rehashidx];
|
||||
// We have to queue every node in the bucket, even if we go over our target size
|
||||
while (de != nullptr) {
|
||||
d->asyncdata->queue.emplace_back(de);
|
||||
de = de->next;
|
||||
}
|
||||
d->rehashidx++;
|
||||
}
|
||||
|
||||
LDone:
|
||||
if (d->asyncdata->queue.empty()) {
|
||||
// We didn't find any work to do (can happen if there is a large gap in the hash table)
|
||||
auto asyncT = d->asyncdata;
|
||||
d->asyncdata = d->asyncdata->next;
|
||||
delete asyncT;
|
||||
return nullptr;
|
||||
}
|
||||
return d->asyncdata;
|
||||
}
|
||||
|
||||
void dictRehashAsync(dictAsyncRehashCtl *ctl) {
|
||||
if (ctl->abondon.load(std::memory_order_acquire)) {
|
||||
ctl->hashIdx = ctl->queue.size();
|
||||
}
|
||||
for (size_t idx = ctl->hashIdx; idx < ctl->queue.size(); ++idx) {
|
||||
auto &wi = ctl->queue[idx];
|
||||
wi.hash = dictHashKey(ctl->dict, dictGetKey(wi.de));
|
||||
}
|
||||
ctl->hashIdx = ctl->queue.size();
|
||||
ctl->done = true;
|
||||
}
|
||||
|
||||
bool dictRehashSomeAsync(dictAsyncRehashCtl *ctl, size_t hashes) {
|
||||
if (ctl->abondon.load(std::memory_order_acquire)) {
|
||||
ctl->hashIdx = ctl->queue.size();
|
||||
}
|
||||
size_t max = std::min(ctl->hashIdx + hashes, ctl->queue.size());
|
||||
for (; ctl->hashIdx < max; ++ctl->hashIdx) {
|
||||
auto &wi = ctl->queue[ctl->hashIdx];
|
||||
wi.hash = dictHashKey(ctl->dict, dictGetKey(wi.de));
|
||||
}
|
||||
|
||||
if (ctl->hashIdx == ctl->queue.size()) ctl->done = true;
|
||||
return ctl->hashIdx < ctl->queue.size();
|
||||
}
|
||||
|
||||
|
||||
void discontinueAsyncRehash(dict *d) {
|
||||
// We inform our async rehashers and the completion function the results are to be
|
||||
// abandoned. We keep the asyncdata linked in so that dictEntry's are still added
|
||||
// to the GC list. This is because we can't gurantee when the other threads will
|
||||
// stop looking at them.
|
||||
if (d->asyncdata != nullptr) {
|
||||
auto adata = d->asyncdata;
|
||||
while (adata != nullptr && !adata->abondon.load(std::memory_order_relaxed)) {
|
||||
adata->abondon = true;
|
||||
adata = adata->next;
|
||||
}
|
||||
if (dictIsRehashing(d))
|
||||
d->rehashidx = 0;
|
||||
}
|
||||
}
|
||||
|
||||
void dictCompleteRehashAsync(dictAsyncRehashCtl *ctl, bool fFree) {
|
||||
dict *d = ctl->dict;
|
||||
assert(ctl->done);
|
||||
|
||||
// Unlink ourselves
|
||||
bool fUnlinked = false;
|
||||
dictAsyncRehashCtl **pprev = &d->asyncdata;
|
||||
|
||||
while (*pprev != nullptr) {
|
||||
if (*pprev == ctl) {
|
||||
*pprev = ctl->next;
|
||||
fUnlinked = true;
|
||||
break;
|
||||
}
|
||||
pprev = &((*pprev)->next);
|
||||
}
|
||||
|
||||
if (fUnlinked) {
|
||||
if (ctl->next != nullptr && ctl->deGCList != nullptr) {
|
||||
// An older work item may depend on our free list, so pass our free list to them
|
||||
dictEntry **deGC = &ctl->next->deGCList;
|
||||
while (*deGC != nullptr) deGC = &((*deGC)->next);
|
||||
*deGC = ctl->deGCList;
|
||||
ctl->deGCList = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
if (fUnlinked && !ctl->abondon) {
|
||||
if (d->ht[0].table != nullptr) { // can be null if we're cleared during the rehash
|
||||
for (auto &wi : ctl->queue) {
|
||||
// We need to remove it from the source hash table, and store it in the dest.
|
||||
// Note that it may have been deleted in the meantime and therefore not exist.
|
||||
// In this case it will be in the garbage collection list
|
||||
|
||||
dictEntry **pdePrev = &d->ht[0].table[wi.hash & d->ht[0].sizemask];
|
||||
while (*pdePrev != nullptr && *pdePrev != wi.de) {
|
||||
pdePrev = &((*pdePrev)->next);
|
||||
}
|
||||
if (*pdePrev != nullptr) { // The element may be NULL if its in the GC list
|
||||
assert(*pdePrev == wi.de);
|
||||
*pdePrev = wi.de->next;
|
||||
// Now link it to the dest hash table
|
||||
wi.de->next = d->ht[1].table[wi.hash & d->ht[1].sizemask];
|
||||
d->ht[1].table[wi.hash & d->ht[1].sizemask] = wi.de;
|
||||
d->ht[0].used--;
|
||||
d->ht[1].used++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Check if we already rehashed the whole table... */
|
||||
if (d->ht[0].used == 0 && d->asyncdata == nullptr) {
|
||||
asyncFreeDictTable(d->ht[0].table);
|
||||
d->ht[0] = d->ht[1];
|
||||
_dictReset(&d->ht[1]);
|
||||
d->rehashidx = -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (fFree) {
|
||||
d->type->asyncfree(ctl);
|
||||
}
|
||||
}
|
||||
|
||||
long long timeInMilliseconds(void) {
|
||||
struct timeval tv;
|
||||
|
||||
@@ -560,18 +256,6 @@ long long timeInMilliseconds(void) {
|
||||
return (((long long)tv.tv_sec)*1000)+(tv.tv_usec/1000);
|
||||
}
|
||||
|
||||
dictAsyncRehashCtl::~dictAsyncRehashCtl() {
|
||||
while (deGCList != nullptr) {
|
||||
auto next = deGCList->next;
|
||||
dictFreeKey(dict, deGCList);
|
||||
if (deGCList->v.val != nullptr)
|
||||
dictFreeVal(dict, deGCList);
|
||||
zfree(deGCList);
|
||||
deGCList = next;
|
||||
}
|
||||
dictRelease(this->dict);
|
||||
}
|
||||
|
||||
/* Rehash in ms+"delta" milliseconds. The value of "delta" is larger
|
||||
* than 0, and is smaller than 1 in most cases. The exact upper bound
|
||||
* depends on the running time of dictRehash(d,100).*/
|
||||
@@ -581,8 +265,8 @@ int dictRehashMilliseconds(dict *d, int ms) {
|
||||
long long start = timeInMilliseconds();
|
||||
int rehashes = 0;
|
||||
|
||||
while(dictRehash(d,1000)) {
|
||||
rehashes += 1000;
|
||||
while(dictRehash(d,100)) {
|
||||
rehashes += 100;
|
||||
if (timeInMilliseconds()-start > ms) break;
|
||||
}
|
||||
return rehashes;
|
||||
@@ -597,15 +281,13 @@ int dictRehashMilliseconds(dict *d, int ms) {
|
||||
* dictionary so that the hash table automatically migrates from H1 to H2
|
||||
* while it is actively used. */
|
||||
static void _dictRehashStep(dict *d) {
|
||||
int16_t pauserehash;
|
||||
__atomic_load(&d->pauserehash, &pauserehash, __ATOMIC_RELAXED);
|
||||
if (pauserehash == 0) dictRehash(d,1);
|
||||
if (d->pauserehash == 0) dictRehash(d,1);
|
||||
}
|
||||
|
||||
/* Add an element to the target hash table */
|
||||
int dictAdd(dict *d, void *key, void *val, dictEntry **existing)
|
||||
int dictAdd(dict *d, void *key, void *val)
|
||||
{
|
||||
dictEntry *entry = dictAddRaw(d,key,existing);
|
||||
dictEntry *entry = dictAddRaw(d,key,NULL);
|
||||
|
||||
if (!entry) return DICT_ERR;
|
||||
dictSetVal(d, entry, val);
|
||||
@@ -707,9 +389,6 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
|
||||
dictEntry *he, *prevHe;
|
||||
int table;
|
||||
|
||||
// if we are deleting elements we probably aren't mass inserting anymore and it is safe to shrink
|
||||
d->noshrink = false;
|
||||
|
||||
if (d->ht[0].used == 0 && d->ht[1].used == 0) return NULL;
|
||||
|
||||
if (dictIsRehashing(d)) _dictRehashStep(d);
|
||||
@@ -727,19 +406,11 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
|
||||
else
|
||||
d->ht[table].table[idx] = he->next;
|
||||
if (!nofree) {
|
||||
if (table == 0 && d->asyncdata != nullptr && (ssize_t)idx < d->rehashidx) {
|
||||
dictFreeVal(d, he);
|
||||
he->v.val = nullptr;
|
||||
he->next = d->asyncdata->deGCList;
|
||||
d->asyncdata->deGCList = he;
|
||||
} else {
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
}
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
}
|
||||
d->ht[table].used--;
|
||||
_dictExpandIfNeeded(d);
|
||||
return he;
|
||||
}
|
||||
prevHe = he;
|
||||
@@ -747,7 +418,6 @@ static dictEntry *dictGenericDelete(dict *d, const void *key, int nofree) {
|
||||
}
|
||||
if (!dictIsRehashing(d)) break;
|
||||
}
|
||||
_dictExpandIfNeeded(d);
|
||||
return NULL; /* not found */
|
||||
}
|
||||
|
||||
@@ -787,16 +457,9 @@ dictEntry *dictUnlink(dict *ht, const void *key) {
|
||||
void dictFreeUnlinkedEntry(dict *d, dictEntry *he) {
|
||||
if (he == NULL) return;
|
||||
|
||||
if (d->asyncdata) {
|
||||
dictFreeVal(d, he);
|
||||
he->v.val = nullptr;
|
||||
he->next = d->asyncdata->deGCList;
|
||||
d->asyncdata->deGCList = he;
|
||||
} else {
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
}
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
}
|
||||
|
||||
/* Destroy an entire dictionary */
|
||||
@@ -810,26 +473,17 @@ int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
|
||||
if (callback && (i & 65535) == 0) callback(d->privdata);
|
||||
|
||||
if ((he = ht->table[i]) == NULL) continue;
|
||||
dictEntry *deNull = nullptr;
|
||||
__atomic_store(&ht->table[i], &deNull, __ATOMIC_RELEASE);
|
||||
while(he) {
|
||||
nextHe = he->next;
|
||||
if (d->asyncdata && (ssize_t)i < d->rehashidx) {
|
||||
dictFreeVal(d, he);
|
||||
he->v.val = nullptr;
|
||||
he->next = d->asyncdata->deGCList;
|
||||
d->asyncdata->deGCList = he;
|
||||
} else {
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
}
|
||||
dictFreeKey(d, he);
|
||||
dictFreeVal(d, he);
|
||||
zfree(he);
|
||||
ht->used--;
|
||||
he = nextHe;
|
||||
}
|
||||
}
|
||||
/* Free the table and the allocated cache structure */
|
||||
asyncFreeDictTable(ht->table);
|
||||
zfree(ht->table);
|
||||
/* Re-initialize the table */
|
||||
_dictReset(ht);
|
||||
return DICT_OK; /* never fails */
|
||||
@@ -838,30 +492,25 @@ int _dictClear(dict *d, dictht *ht, void(callback)(void *)) {
|
||||
/* Clear & Release the hash table */
|
||||
void dictRelease(dict *d)
|
||||
{
|
||||
if (__atomic_sub_fetch(&d->refcount, 1, __ATOMIC_ACQ_REL) == 0) {
|
||||
_dictClear(d,&d->ht[0],NULL);
|
||||
_dictClear(d,&d->ht[1],NULL);
|
||||
zfree(d);
|
||||
}
|
||||
_dictClear(d,&d->ht[0],NULL);
|
||||
_dictClear(d,&d->ht[1],NULL);
|
||||
zfree(d);
|
||||
}
|
||||
|
||||
dictEntry *dictFindWithPrev(dict *d, const void *key, uint64_t h, dictEntry ***dePrevPtr, dictht **pht, bool fShallowCompare)
|
||||
dictEntry *dictFind(dict *d, const void *key)
|
||||
{
|
||||
dictEntry *he;
|
||||
uint64_t idx, table;
|
||||
uint64_t h, idx, table;
|
||||
|
||||
if (dictSize(d) == 0) return NULL; /* dict is empty */
|
||||
if (dictIsRehashing(d)) _dictRehashStep(d);
|
||||
h = dictHashKey(d, key);
|
||||
for (table = 0; table <= 1; table++) {
|
||||
*pht = d->ht + table;
|
||||
idx = h & d->ht[table].sizemask;
|
||||
he = d->ht[table].table[idx];
|
||||
*dePrevPtr = &d->ht[table].table[idx];
|
||||
while(he) {
|
||||
if (key==he->key || (!fShallowCompare && dictCompareKeys(d, key, he->key))) {
|
||||
if (key==he->key || dictCompareKeys(d, key, he->key))
|
||||
return he;
|
||||
}
|
||||
*dePrevPtr = &he->next;
|
||||
he = he->next;
|
||||
}
|
||||
if (!dictIsRehashing(d)) return NULL;
|
||||
@@ -869,14 +518,6 @@ dictEntry *dictFindWithPrev(dict *d, const void *key, uint64_t h, dictEntry ***d
|
||||
return NULL;
|
||||
}
|
||||
|
||||
dictEntry *dictFind(dict *d, const void *key)
|
||||
{
|
||||
dictEntry **deT;
|
||||
dictht *ht;
|
||||
uint64_t h = dictHashKey(d, key);
|
||||
return dictFindWithPrev(d, key, h, &deT, &ht);
|
||||
}
|
||||
|
||||
void *dictFetchValue(dict *d, const void *key) {
|
||||
dictEntry *he;
|
||||
|
||||
@@ -999,16 +640,10 @@ dictEntry *dictGetRandomKey(dict *d)
|
||||
if (dictSize(d) == 0) return NULL;
|
||||
if (dictIsRehashing(d)) _dictRehashStep(d);
|
||||
if (dictIsRehashing(d)) {
|
||||
long rehashidx = d->rehashidx;
|
||||
auto async = d->asyncdata;
|
||||
while (async != nullptr) {
|
||||
rehashidx = std::min((long)async->rehashIdxBase, rehashidx);
|
||||
async = async->next;
|
||||
}
|
||||
do {
|
||||
/* We are sure there are no elements in indexes from 0
|
||||
* to rehashidx-1 */
|
||||
h = rehashidx + (randomULong() % (dictSlots(d) - rehashidx));
|
||||
h = d->rehashidx + (randomULong() % (dictSlots(d) - d->rehashidx));
|
||||
he = (h >= d->ht[0].size) ? d->ht[1].table[h - d->ht[0].size] :
|
||||
d->ht[0].table[h];
|
||||
} while(he == NULL);
|
||||
@@ -1347,12 +982,11 @@ static int dictTypeExpandAllowed(dict *d) {
|
||||
/* Expand the hash table if needed */
|
||||
static int _dictExpandIfNeeded(dict *d)
|
||||
{
|
||||
static const size_t SHRINK_FACTOR = 4;
|
||||
/* Incremental rehashing already in progress. Return. */
|
||||
if (dictIsRehashing(d)) return DICT_OK;
|
||||
|
||||
/* If the hash table is empty expand it to the initial size. */
|
||||
if (d->ht[0].size == 0) return _dictExpand(d, DICT_HT_INITIAL_SIZE, false /*fShrink*/, nullptr);
|
||||
if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE);
|
||||
|
||||
/* If we reached the 1:1 ratio, and we are allowed to resize the hash
|
||||
* table (global setting) or we should avoid it but the ratio between
|
||||
@@ -1363,12 +997,7 @@ static int _dictExpandIfNeeded(dict *d)
|
||||
d->ht[0].used/d->ht[0].size > dict_force_resize_ratio) &&
|
||||
dictTypeExpandAllowed(d))
|
||||
{
|
||||
return _dictExpand(d, d->ht[0].used + 1, false /*fShrink*/, nullptr);
|
||||
}
|
||||
else if (d->ht[0].used > 0 && d->ht[0].size >= (1024*SHRINK_FACTOR) && (d->ht[0].used * 16) < d->ht[0].size && dict_can_resize && !d->noshrink)
|
||||
{
|
||||
// If the dictionary has shurnk a lot we'll need to shrink the hash table instead
|
||||
return _dictExpand(d, d->ht[0].size/SHRINK_FACTOR, true /*fShrink*/, nullptr);
|
||||
return dictExpand(d, d->ht[0].used + 1);
|
||||
}
|
||||
return DICT_OK;
|
||||
}
|
||||
@@ -1393,7 +1022,7 @@ static unsigned long _dictNextPower(unsigned long size)
|
||||
*
|
||||
* Note that if we are in the process of rehashing the hash table, the
|
||||
* index is always returned in the context of the second (new) hash table. */
|
||||
long _dictKeyIndex(dict *d, const void *key, uint64_t hash, dictEntry **existing)
|
||||
static long _dictKeyIndex(dict *d, const void *key, uint64_t hash, dictEntry **existing)
|
||||
{
|
||||
unsigned long idx, table;
|
||||
dictEntry *he;
|
||||
@@ -1541,13 +1170,6 @@ void dictGetStats(char *buf, size_t bufsize, dict *d) {
|
||||
if (orig_bufsize) orig_buf[orig_bufsize-1] = '\0';
|
||||
}
|
||||
|
||||
void dictForceRehash(dict *d)
|
||||
{
|
||||
int16_t pauserehash;
|
||||
__atomic_load(&d->pauserehash, &pauserehash, __ATOMIC_RELAXED);
|
||||
while (pauserehash == 0 && dictIsRehashing(d)) _dictRehashStep(d);
|
||||
}
|
||||
|
||||
/* ------------------------------- Benchmark ---------------------------------*/
|
||||
|
||||
#ifdef REDIS_TEST
|
||||
@@ -1577,7 +1199,7 @@ char *stringFromLongLong(long long value) {
|
||||
int len;
|
||||
char *s;
|
||||
|
||||
len = snprintf(buf,32,"%lld",value);
|
||||
len = sprintf(buf,"%lld",value);
|
||||
s = zmalloc(len+1);
|
||||
memcpy(s, buf, len);
|
||||
s[len] = '\0';
|
||||
|
||||
+9
-45
@@ -56,7 +56,6 @@ extern "C" {
|
||||
|
||||
/* Unused arguments generate annoying warnings... */
|
||||
#define DICT_NOTUSED(V) ((void) V)
|
||||
struct dictAsyncRehashCtl;
|
||||
|
||||
typedef struct dictEntry {
|
||||
void *key;
|
||||
@@ -77,7 +76,6 @@ typedef struct dictType {
|
||||
void (*keyDestructor)(void *privdata, void *key);
|
||||
void (*valDestructor)(void *privdata, void *obj);
|
||||
int (*expandAllowed)(size_t moreMem, double usedRatio);
|
||||
void (*asyncfree)(dictAsyncRehashCtl *);
|
||||
} dictType;
|
||||
|
||||
/* This is our hash table structure. Every dictionary has two of this as we
|
||||
@@ -104,49 +102,24 @@ struct dictAsyncRehashCtl {
|
||||
struct dict *dict = nullptr;
|
||||
std::vector<workItem> queue;
|
||||
size_t hashIdx = 0;
|
||||
long rehashIdxBase;
|
||||
bool release = false;
|
||||
dictAsyncRehashCtl *next = nullptr;
|
||||
std::atomic<bool> done { false };
|
||||
std::atomic<bool> abondon { false };
|
||||
|
||||
dictAsyncRehashCtl(struct dict *d, dictAsyncRehashCtl *next);
|
||||
dictAsyncRehashCtl(const dictAsyncRehashCtl&) = delete;
|
||||
dictAsyncRehashCtl(dictAsyncRehashCtl&&) = delete;
|
||||
~dictAsyncRehashCtl();
|
||||
dictAsyncRehashCtl(struct dict *d, dictAsyncRehashCtl *next) : dict(d), next(next) {
|
||||
queue.reserve(c_targetQueueSize);
|
||||
}
|
||||
};
|
||||
#else
|
||||
struct dictAsyncRehashCtl;
|
||||
#endif
|
||||
|
||||
void discontinueAsyncRehash(dict *d);
|
||||
|
||||
typedef struct dict {
|
||||
dictType *type;
|
||||
void *privdata;
|
||||
dictht ht[2];
|
||||
long rehashidx; /* rehashing not in progress if rehashidx == -1 */
|
||||
unsigned refcount;
|
||||
dictAsyncRehashCtl *asyncdata;
|
||||
int16_t pauserehash; /* If >0 rehashing is paused (<0 indicates coding error) */
|
||||
uint8_t noshrink = false;
|
||||
|
||||
#ifdef __cplusplus
|
||||
dict() = default;
|
||||
dict(dict &) = delete; // No Copy Ctor
|
||||
|
||||
static void swap(dict& a, dict& b) {
|
||||
discontinueAsyncRehash(&a);
|
||||
discontinueAsyncRehash(&b);
|
||||
std::swap(a.type, b.type);
|
||||
std::swap(a.privdata, b.privdata);
|
||||
std::swap(a.ht[0], b.ht[0]);
|
||||
std::swap(a.ht[1], b.ht[1]);
|
||||
std::swap(a.rehashidx, b.rehashidx);
|
||||
// Never swap refcount - they are attached to the specific dict obj
|
||||
std::swap(a.pauserehash, b.pauserehash);
|
||||
std::swap(a.noshrink, b.noshrink);
|
||||
}
|
||||
#endif
|
||||
} dict;
|
||||
|
||||
/* If safe is set to 1 this is a safe iterator, that means, you can call
|
||||
@@ -214,8 +187,8 @@ typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
|
||||
#define dictSlots(d) ((d)->ht[0].size+(d)->ht[1].size)
|
||||
#define dictSize(d) ((d)->ht[0].used+(d)->ht[1].used)
|
||||
#define dictIsRehashing(d) ((d)->rehashidx != -1)
|
||||
#define dictPauseRehashing(d) __atomic_fetch_add(&(d)->pauserehash, 1, __ATOMIC_SEQ_CST)
|
||||
#define dictResumeRehashing(d) __atomic_fetch_sub(&(d)->pauserehash, 1, __ATOMIC_SEQ_CST)
|
||||
#define dictPauseRehashing(d) (d)->pauserehash++
|
||||
#define dictResumeRehashing(d) (d)->pauserehash--
|
||||
|
||||
/* If our unsigned long type can store a 64 bit number, use a 64 bit PRNG. */
|
||||
#if ULONG_MAX >= 0xffffffffffffffff
|
||||
@@ -226,9 +199,9 @@ typedef void (dictScanBucketFunction)(void *privdata, dictEntry **bucketref);
|
||||
|
||||
/* API */
|
||||
dict *dictCreate(dictType *type, void *privDataPtr);
|
||||
int dictExpand(dict *d, unsigned long size, bool fShrink = false);
|
||||
int dictTryExpand(dict *d, unsigned long size, bool fShrink);
|
||||
int dictAdd(dict *d, void *key, void *val, dictEntry **existing = nullptr);
|
||||
int dictExpand(dict *d, unsigned long size);
|
||||
int dictTryExpand(dict *d, unsigned long size);
|
||||
int dictAdd(dict *d, void *key, void *val);
|
||||
dictEntry *dictAddRaw(dict *d, void *key, dictEntry **existing);
|
||||
dictEntry *dictAddOrFind(dict *d, void *key);
|
||||
int dictReplace(dict *d, void *key, void *val);
|
||||
@@ -237,7 +210,6 @@ dictEntry *dictUnlink(dict *ht, const void *key);
|
||||
void dictFreeUnlinkedEntry(dict *d, dictEntry *he);
|
||||
void dictRelease(dict *d);
|
||||
dictEntry * dictFind(dict *d, const void *key);
|
||||
dictEntry * dictFindWithPrev(dict *d, const void *key, uint64_t h, dictEntry ***dePrevPtr, dictht **ht, bool fShallowCompare = false);
|
||||
void *dictFetchValue(dict *d, const void *key);
|
||||
int dictResize(dict *d);
|
||||
dictIterator *dictGetIterator(dict *d);
|
||||
@@ -260,14 +232,6 @@ uint8_t *dictGetHashFunctionSeed(void);
|
||||
unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, dictScanBucketFunction *bucketfn, void *privdata);
|
||||
uint64_t dictGetHash(dict *d, const void *key);
|
||||
dictEntry **dictFindEntryRefByPtrAndHash(dict *d, const void *oldptr, uint64_t hash);
|
||||
void dictForceRehash(dict *d);
|
||||
int dictMerge(dict *dst, dict *src);
|
||||
|
||||
/* Async API */
|
||||
dictAsyncRehashCtl *dictRehashAsyncStart(dict *d, int buckets);
|
||||
bool dictRehashSomeAsync(dictAsyncRehashCtl *ctl, size_t hashes);
|
||||
void dictCompleteRehashAsync(dictAsyncRehashCtl *ctl, bool fFree);
|
||||
void dictRehashAsync(dictAsyncRehashCtl *ctl);
|
||||
|
||||
/* Hash table types */
|
||||
extern dictType dictTypeHeapStringCopyKey;
|
||||
|
||||
+3
-3
@@ -112,15 +112,15 @@ int endianconvTest(int argc, char *argv[], int accurate) {
|
||||
UNUSED(argv);
|
||||
UNUSED(accurate);
|
||||
|
||||
snprintf(buf,32,"ciaoroma");
|
||||
sprintf(buf,"ciaoroma");
|
||||
memrev16(buf);
|
||||
printf("%s\n", buf);
|
||||
|
||||
snprintf(buf,32,"ciaoroma");
|
||||
sprintf(buf,"ciaoroma");
|
||||
memrev32(buf);
|
||||
printf("%s\n", buf);
|
||||
|
||||
snprintf(buf,32,"ciaoroma");
|
||||
sprintf(buf,"ciaoroma");
|
||||
memrev64(buf);
|
||||
printf("%s\n", buf);
|
||||
|
||||
|
||||
+71
-231
@@ -34,7 +34,6 @@
|
||||
#include "bio.h"
|
||||
#include "atomicvar.h"
|
||||
#include <mutex>
|
||||
#include <map>
|
||||
#include <math.h>
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
@@ -227,47 +226,32 @@ struct visitFunctor
|
||||
int dbid;
|
||||
dict *dbdict;
|
||||
struct evictionPoolEntry *pool;
|
||||
int count = 0;
|
||||
int tries = 0;
|
||||
int count;
|
||||
|
||||
bool operator()(const expireEntry &e)
|
||||
{
|
||||
dictEntry *de = dictFind(dbdict, e.key());
|
||||
if (de != nullptr)
|
||||
{
|
||||
processEvictionCandidate(dbid, (sds)dictGetKey(de), (robj*)dictGetVal(de), &e, pool);
|
||||
++count;
|
||||
}
|
||||
++tries;
|
||||
return tries < g_pserver->maxmemory_samples;
|
||||
processEvictionCandidate(dbid, (sds)dictGetKey(de), (robj*)dictGetVal(de), &e, pool);
|
||||
++count;
|
||||
return count < g_pserver->maxmemory_samples;
|
||||
}
|
||||
};
|
||||
int evictionPoolPopulate(int dbid, redisDb *db, expireset *setexpire, struct evictionPoolEntry *pool)
|
||||
void evictionPoolPopulate(int dbid, dict *dbdict, expireset *setexpire, struct evictionPoolEntry *pool)
|
||||
{
|
||||
if (setexpire != nullptr)
|
||||
{
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
visitFunctor visitor { dbid, db->dictUnsafeKeyOnly(), pool, 0 };
|
||||
visitFunctor visitor { dbid, dbdict, pool, 0 };
|
||||
setexpire->random_visit(visitor);
|
||||
return visitor.count;
|
||||
}
|
||||
else
|
||||
{
|
||||
int returnCount = 0;
|
||||
dictEntry **samples = (dictEntry**)alloca(g_pserver->maxmemory_samples * sizeof(dictEntry*));
|
||||
int count = dictGetSomeKeys(db->dictUnsafeKeyOnly(),samples,g_pserver->maxmemory_samples);
|
||||
int count = dictGetSomeKeys(dbdict,samples,g_pserver->maxmemory_samples);
|
||||
for (int j = 0; j < count; j++) {
|
||||
robj *o = (robj*)dictGetVal(samples[j]);
|
||||
// If the object is in second tier storage we don't need to evict it (since it alrady is)
|
||||
if (o != nullptr)
|
||||
{
|
||||
processEvictionCandidate(dbid, (sds)dictGetKey(samples[j]), o, nullptr, pool);
|
||||
++returnCount;
|
||||
}
|
||||
processEvictionCandidate(dbid, (sds)dictGetKey(samples[j]), o, nullptr, pool);
|
||||
}
|
||||
return returnCount;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ----------------------------------------------------------------------------
|
||||
@@ -346,7 +330,7 @@ uint8_t LFULogIncr(uint8_t counter) {
|
||||
* This function is used in order to scan the dataset for the best object
|
||||
* to fit: as we check for the candidate, we incrementally decrement the
|
||||
* counter of the scanned objects if needed. */
|
||||
unsigned long LFUDecrAndReturn(robj_roptr o) {
|
||||
unsigned long LFUDecrAndReturn(robj *o) {
|
||||
unsigned long ldt = o->lru >> 8;
|
||||
unsigned long counter = o->lru & 255;
|
||||
unsigned long num_periods = g_pserver->lfu_decay_time ? LFUTimeElapsed(ldt) / g_pserver->lfu_decay_time : 0;
|
||||
@@ -355,8 +339,6 @@ unsigned long LFUDecrAndReturn(robj_roptr o) {
|
||||
return counter;
|
||||
}
|
||||
|
||||
unsigned long getClientReplicationBacklogSharedUsage(client *c);
|
||||
|
||||
/* We don't want to count AOF buffers and slaves output buffers as
|
||||
* used memory: the eviction should use mostly data size. This function
|
||||
* returns the sum of AOF and slaves buffer. */
|
||||
@@ -373,15 +355,9 @@ size_t freeMemoryGetNotCountedMemory(void) {
|
||||
while((ln = listNext(&li))) {
|
||||
client *replica = (client*)listNodeValue(ln);
|
||||
std::unique_lock<fastlock>(replica->lock);
|
||||
/* we don't wish to multiple count the replication backlog shared usage */
|
||||
overhead += (getClientOutputBufferMemoryUsage(replica) - getClientReplicationBacklogSharedUsage(replica));
|
||||
overhead += getClientOutputBufferMemoryUsage(replica);
|
||||
}
|
||||
}
|
||||
|
||||
/* also don't count the replication backlog memory
|
||||
* that's where the replication clients get their memory from */
|
||||
overhead += g_pserver->repl_backlog_size;
|
||||
|
||||
if (g_pserver->aof_state != AOF_OFF) {
|
||||
overhead += sdsalloc(g_pserver->aof_buf)+aofRewriteBufferSize();
|
||||
}
|
||||
@@ -412,21 +388,16 @@ size_t freeMemoryGetNotCountedMemory(void) {
|
||||
* limit.
|
||||
* (Populated both for C_ERR and C_OK)
|
||||
*/
|
||||
int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level, bool fQuickCycle, bool fPreSnapshot) {
|
||||
int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *level) {
|
||||
size_t mem_reported, mem_used, mem_tofree;
|
||||
|
||||
/* Check if we are over the memory usage limit. If we are not, no need
|
||||
* to subtract the slaves output buffers. We can just return ASAP. */
|
||||
mem_reported = zmalloc_used_memory();
|
||||
if (total) *total = mem_reported;
|
||||
size_t maxmemory = g_pserver->maxmemory;
|
||||
if (fPreSnapshot)
|
||||
maxmemory = static_cast<size_t>(maxmemory * 0.9); // derate memory by 10% since we won't be able to free during snapshot
|
||||
if (g_pserver->FRdbSaveInProgress())
|
||||
maxmemory = static_cast<size_t>(maxmemory*1.2);
|
||||
|
||||
/* We may return ASAP if there is no need to compute the level. */
|
||||
int return_ok_asap = !maxmemory || mem_reported <= maxmemory;
|
||||
int return_ok_asap = !g_pserver->maxmemory || mem_reported <= g_pserver->maxmemory;
|
||||
if (return_ok_asap && !level) return C_OK;
|
||||
|
||||
/* Remove the size of slaves output buffers and AOF buffer from the
|
||||
@@ -437,24 +408,20 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
|
||||
|
||||
/* Compute the ratio of memory usage. */
|
||||
if (level) {
|
||||
if (!maxmemory) {
|
||||
if (!g_pserver->maxmemory) {
|
||||
*level = 0;
|
||||
} else {
|
||||
*level = (float)mem_used / (float)maxmemory;
|
||||
*level = (float)mem_used / (float)g_pserver->maxmemory;
|
||||
}
|
||||
}
|
||||
|
||||
if (return_ok_asap) return C_OK;
|
||||
|
||||
/* Check if we are still over the memory limit. */
|
||||
if (mem_used <= maxmemory) return C_OK;
|
||||
if (mem_used <= g_pserver->maxmemory) return C_OK;
|
||||
|
||||
/* Compute how much memory we need to free. */
|
||||
mem_tofree = mem_used - maxmemory;
|
||||
if (g_pserver->m_pstorageFactory && !fQuickCycle)
|
||||
{
|
||||
mem_tofree += static_cast<size_t>(maxmemory * 0.05); // if we have a storage provider be much more aggressive
|
||||
}
|
||||
mem_tofree = mem_used - g_pserver->maxmemory;
|
||||
|
||||
if (logical) *logical = mem_used;
|
||||
if (tofree) *tofree = mem_tofree;
|
||||
@@ -462,69 +429,6 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev
|
||||
return C_ERR;
|
||||
}
|
||||
|
||||
class FreeMemoryLazyFree : public ICollectable
|
||||
{
|
||||
ssize_t m_cb = 0;
|
||||
std::vector<std::pair<dict*, std::vector<dictEntry*>>> vecdictvecde;
|
||||
|
||||
public:
|
||||
static std::atomic<int> s_clazyFreesInProgress;
|
||||
|
||||
FreeMemoryLazyFree() {
|
||||
s_clazyFreesInProgress++;
|
||||
}
|
||||
|
||||
FreeMemoryLazyFree(const FreeMemoryLazyFree&) = delete;
|
||||
FreeMemoryLazyFree(FreeMemoryLazyFree&&) = default;
|
||||
|
||||
~FreeMemoryLazyFree() {
|
||||
aeAcquireLock();
|
||||
for (auto &pair : vecdictvecde) {
|
||||
for (auto de : pair.second) {
|
||||
dictFreeUnlinkedEntry(pair.first, de);
|
||||
}
|
||||
}
|
||||
aeReleaseLock();
|
||||
--s_clazyFreesInProgress;
|
||||
}
|
||||
|
||||
ssize_t addEntry(dict *d, dictEntry *de) {
|
||||
ssize_t cbFreedNow = 0;
|
||||
ssize_t cb = sizeof(dictEntry);
|
||||
cb += sdsAllocSize((sds)dictGetKey(de));
|
||||
robj *o = (robj*)dictGetVal(de);
|
||||
switch (o->type) {
|
||||
case OBJ_STRING:
|
||||
cb += getStringObjectSdsUsedMemory(o)+sizeof(robj);
|
||||
break;
|
||||
|
||||
default:
|
||||
// If we don't know about it we can't accurately track the memory so free now
|
||||
cbFreedNow = zmalloc_used_memory();
|
||||
decrRefCount(o);
|
||||
cbFreedNow -= zmalloc_used_memory();
|
||||
de->v.val = nullptr;
|
||||
}
|
||||
|
||||
auto itr = std::lower_bound(vecdictvecde.begin(), vecdictvecde.end(), d,
|
||||
[](const std::pair<dict*, std::vector<dictEntry*>> &a, dict *d) -> bool {
|
||||
return a.first < d;
|
||||
}
|
||||
);
|
||||
if (itr == vecdictvecde.end() || itr->first != d) {
|
||||
itr = vecdictvecde.insert(itr, std::make_pair(d, std::vector<dictEntry*>()));
|
||||
}
|
||||
serverAssert(itr->first == d);
|
||||
itr->second.push_back(de);
|
||||
m_cb += cb;
|
||||
return cb + cbFreedNow;
|
||||
}
|
||||
|
||||
size_t memory_queued() { return m_cb; }
|
||||
};
|
||||
|
||||
std::atomic<int> FreeMemoryLazyFree::s_clazyFreesInProgress {0};
|
||||
|
||||
/* Return 1 if used memory is more than maxmemory after allocating more memory,
|
||||
* return 0 if not. Redis may reject user's requests or evict some keys if used
|
||||
* memory exceeds maxmemory, especially, when we allocate huge memory at once. */
|
||||
@@ -550,9 +454,8 @@ static int evictionTimeProc(
|
||||
UNUSED(eventLoop);
|
||||
UNUSED(id);
|
||||
UNUSED(clientData);
|
||||
serverAssert(GlobalLocksAcquired());
|
||||
|
||||
if (performEvictions(false) == EVICT_RUNNING) return 0; /* keep evicting */
|
||||
if (performEvictions() == EVICT_RUNNING) return 0; /* keep evicting */
|
||||
|
||||
/* For EVICT_OK - things are good, no need to keep evicting.
|
||||
* For EVICT_FAIL - there is nothing left to evict. */
|
||||
@@ -567,14 +470,11 @@ static int evictionTimeProc(
|
||||
static int isSafeToPerformEvictions(void) {
|
||||
/* - There must be no script in timeout condition.
|
||||
* - Nor we are loading data right now. */
|
||||
if (g_pserver->shutdown_asap || g_pserver->lua_timedout || g_pserver->loading) return 0;
|
||||
if (g_pserver->lua_timedout || g_pserver->loading) return 0;
|
||||
|
||||
/* By default replicas should ignore maxmemory
|
||||
* and just be masters exact copies. */
|
||||
if (g_pserver->m_pstorageFactory == nullptr && listLength(g_pserver->masters) && g_pserver->repl_slave_ignore_maxmemory && !g_pserver->fActiveReplica) return 0;
|
||||
|
||||
/* If we have a lazy free obj pending, our amounts will be off, wait for it to go away */
|
||||
if (FreeMemoryLazyFree::s_clazyFreesInProgress > 0) return 0;
|
||||
if (listLength(g_pserver->masters) && g_pserver->repl_slave_ignore_maxmemory && !g_pserver->fActiveReplica) return 0;
|
||||
|
||||
/* When clients are paused the dataset should be static not just from the
|
||||
* POV of clients not being able to write, but also from the POV of
|
||||
@@ -626,7 +526,7 @@ static unsigned long evictionTimeLimitUs() {
|
||||
* EVICT_RUNNING - memory is over the limit, but eviction is still processing
|
||||
* EVICT_FAIL - memory is over the limit, and there's nothing to evict
|
||||
* */
|
||||
int performEvictions(bool fPreSnapshot) {
|
||||
int performEvictions(void) {
|
||||
if (!isSafeToPerformEvictions()) return EVICT_OK;
|
||||
serverAssert(GlobalLocksAcquired());
|
||||
|
||||
@@ -636,13 +536,9 @@ int performEvictions(bool fPreSnapshot) {
|
||||
mstime_t latency, eviction_latency;
|
||||
long long delta;
|
||||
int slaves = listLength(g_pserver->slaves);
|
||||
const bool fEvictToStorage = !cserver.delete_on_evict && g_pserver->db[0]->FStorageProvider();
|
||||
int result = EVICT_FAIL;
|
||||
int ckeysFailed = 0;
|
||||
|
||||
std::unique_ptr<FreeMemoryLazyFree> splazy = std::make_unique<FreeMemoryLazyFree>();
|
||||
|
||||
if (getMaxmemoryState(&mem_reported,NULL,&mem_tofree,NULL,false,fPreSnapshot) == C_OK)
|
||||
if (getMaxmemoryState(&mem_reported,NULL,&mem_tofree,NULL) == C_OK)
|
||||
return EVICT_OK;
|
||||
|
||||
if (g_pserver->maxmemory_policy == MAXMEMORY_NO_EVICTION)
|
||||
@@ -657,17 +553,13 @@ int performEvictions(bool fPreSnapshot) {
|
||||
monotime evictionTimer;
|
||||
elapsedStart(&evictionTimer);
|
||||
|
||||
if (g_pserver->maxstorage && g_pserver->m_pstorageFactory != nullptr && g_pserver->m_pstorageFactory->totalDiskspaceUsed() >= g_pserver->maxstorage)
|
||||
goto cant_free_storage;
|
||||
|
||||
while (mem_freed < (long long)mem_tofree) {
|
||||
int j, k, i;
|
||||
static unsigned int next_db = 0;
|
||||
sds bestkey = NULL;
|
||||
int bestdbid;
|
||||
redisDb *db;
|
||||
bool fFallback = false;
|
||||
|
||||
|
||||
if (g_pserver->maxmemory_policy & (MAXMEMORY_FLAG_LRU|MAXMEMORY_FLAG_LFU) ||
|
||||
g_pserver->maxmemory_policy == MAXMEMORY_VOLATILE_TTL)
|
||||
{
|
||||
@@ -680,33 +572,33 @@ int performEvictions(bool fPreSnapshot) {
|
||||
* so to start populate the eviction pool sampling keys from
|
||||
* every DB. */
|
||||
for (i = 0; i < cserver.dbnum; i++) {
|
||||
db = g_pserver->db[i];
|
||||
db = g_pserver->db+i;
|
||||
if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS)
|
||||
{
|
||||
if ((keys = db->size()) != 0) {
|
||||
total_keys += evictionPoolPopulate(i, db, nullptr, pool);
|
||||
if ((keys = dictSize(db->dict)) != 0) {
|
||||
evictionPoolPopulate(i, db->dict, nullptr, pool);
|
||||
total_keys += keys;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
keys = db->expireSize();
|
||||
keys = db->setexpire->size();
|
||||
if (keys != 0)
|
||||
total_keys += evictionPoolPopulate(i, db, db->setexpireUnsafe(), pool);
|
||||
evictionPoolPopulate(i, db->dict, db->setexpire, pool);
|
||||
total_keys += keys;
|
||||
}
|
||||
}
|
||||
if (!total_keys) break; /* No keys to evict. */
|
||||
|
||||
/* Go backward from best to worst element to evict. */
|
||||
for (k = EVPOOL_SIZE-1; k >= 0; k--) {
|
||||
if (pool[k].key == NULL) {
|
||||
continue;
|
||||
}
|
||||
if (pool[k].key == NULL) continue;
|
||||
bestdbid = pool[k].dbid;
|
||||
sds key = nullptr;
|
||||
|
||||
auto itr = g_pserver->db[pool[k].dbid]->find(pool[k].key);
|
||||
if (itr != nullptr && (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS || itr.val()->FExpires()))
|
||||
key = itr.key();
|
||||
dictEntry *de = dictFind(g_pserver->db[pool[k].dbid].dict,pool[k].key);
|
||||
if (de != nullptr && (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS || ((robj*)dictGetVal(de))->FExpires()))
|
||||
key = (sds)dictGetKey(de);
|
||||
|
||||
/* Remove the entry from the pool. */
|
||||
if (pool[k].key != pool[k].cached)
|
||||
@@ -724,35 +616,32 @@ int performEvictions(bool fPreSnapshot) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (bestkey == nullptr && fEvictToStorage)
|
||||
fFallback = true;
|
||||
}
|
||||
|
||||
/* volatile-random and allkeys-random policy */
|
||||
if (g_pserver->maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM ||
|
||||
g_pserver->maxmemory_policy == MAXMEMORY_VOLATILE_RANDOM
|
||||
|| fFallback)
|
||||
else if (g_pserver->maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM ||
|
||||
g_pserver->maxmemory_policy == MAXMEMORY_VOLATILE_RANDOM)
|
||||
{
|
||||
/* When evicting a random key, we try to evict a key for
|
||||
* each DB, so we use the static 'next_db' variable to
|
||||
* incrementally visit all DBs. */
|
||||
for (i = 0; i < cserver.dbnum; i++) {
|
||||
j = (++next_db) % cserver.dbnum;
|
||||
db = g_pserver->db[j];
|
||||
if (g_pserver->maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM || fFallback)
|
||||
db = g_pserver->db+j;
|
||||
if (g_pserver->maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM)
|
||||
{
|
||||
if (db->size() != 0) {
|
||||
auto itr = db->random_cache_threadsafe(true /*fPrimaryOnly*/); // primary only because we can't evict a snapshot key
|
||||
bestkey = itr.key();
|
||||
if (dictSize(db->dict) != 0) {
|
||||
dictEntry *de = dictGetRandomKey(db->dict);
|
||||
bestkey = (sds)dictGetKey(de);
|
||||
bestdbid = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (db->expireSize())
|
||||
if (!db->setexpire->empty())
|
||||
{
|
||||
bestkey = (sds)db->random_expire().key();
|
||||
bestkey = (sds)db->setexpire->random_value().key();
|
||||
bestdbid = j;
|
||||
break;
|
||||
}
|
||||
@@ -762,51 +651,34 @@ int performEvictions(bool fPreSnapshot) {
|
||||
|
||||
/* Finally remove the selected key. */
|
||||
if (bestkey) {
|
||||
db = g_pserver->db[bestdbid];
|
||||
|
||||
if (fEvictToStorage)
|
||||
{
|
||||
// This key is in the storage so we only need to free the object
|
||||
dictEntry *deT;
|
||||
if (db->removeCachedValue(bestkey, &deT)) {
|
||||
mem_freed += splazy->addEntry(db->dictUnsafeKeyOnly(), deT);
|
||||
ckeysFailed = 0;
|
||||
}
|
||||
else {
|
||||
delta = 0;
|
||||
ckeysFailed++;
|
||||
if (ckeysFailed > 1024)
|
||||
goto cant_free;
|
||||
}
|
||||
}
|
||||
db = g_pserver->db+bestdbid;
|
||||
robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
|
||||
propagateExpire(db,keyobj,g_pserver->lazyfree_lazy_eviction);
|
||||
/* We compute the amount of memory freed by db*Delete() alone.
|
||||
* It is possible that actually the memory needed to propagate
|
||||
* the DEL in AOF and replication link is greater than the one
|
||||
* we are freeing removing the key, but we can't account for
|
||||
* that otherwise we would never exit the loop.
|
||||
*
|
||||
* Same for CSC invalidation messages generated by signalModifiedKey.
|
||||
*
|
||||
* AOF and Output buffer memory will be freed eventually so
|
||||
* we only care about memory used by the key space. */
|
||||
delta = (long long) zmalloc_used_memory();
|
||||
latencyStartMonitor(eviction_latency);
|
||||
if (g_pserver->lazyfree_lazy_eviction)
|
||||
dbAsyncDelete(db,keyobj);
|
||||
else
|
||||
{
|
||||
robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
|
||||
propagateExpire(db,keyobj,g_pserver->lazyfree_lazy_eviction);
|
||||
/* We compute the amount of memory freed by db*Delete() alone.
|
||||
* It is possible that actually the memory needed to propagate
|
||||
* the DEL in AOF and replication link is greater than the one
|
||||
* we are freeing removing the key, but we can't account for
|
||||
* that otherwise we would never exit the loop.
|
||||
*
|
||||
* AOF and Output buffer memory will be freed eventually so
|
||||
* we only care about memory used by the key space. */
|
||||
delta = (long long) zmalloc_used_memory();
|
||||
latencyStartMonitor(eviction_latency);
|
||||
if (g_pserver->lazyfree_lazy_eviction)
|
||||
dbAsyncDelete(db,keyobj);
|
||||
else
|
||||
dbSyncDelete(db,keyobj);
|
||||
latencyEndMonitor(eviction_latency);
|
||||
latencyAddSampleIfNeeded("eviction-del",eviction_latency);
|
||||
delta -= (long long) zmalloc_used_memory();
|
||||
mem_freed += delta;
|
||||
g_pserver->stat_evictedkeys++;
|
||||
signalModifiedKey(NULL,db,keyobj);
|
||||
notifyKeyspaceEvent(NOTIFY_EVICTED, "evicted",
|
||||
keyobj, db->id);
|
||||
decrRefCount(keyobj);
|
||||
}
|
||||
dbSyncDelete(db,keyobj);
|
||||
latencyEndMonitor(eviction_latency);
|
||||
latencyAddSampleIfNeeded("eviction-del",eviction_latency);
|
||||
delta -= (long long) zmalloc_used_memory();
|
||||
mem_freed += delta;
|
||||
g_pserver->stat_evictedkeys++;
|
||||
signalModifiedKey(NULL,db,keyobj);
|
||||
notifyKeyspaceEvent(NOTIFY_EVICTED, "evicted",
|
||||
keyobj, db->id);
|
||||
decrRefCount(keyobj);
|
||||
keys_freed++;
|
||||
|
||||
if (keys_freed % 16 == 0) {
|
||||
@@ -832,9 +704,9 @@ int performEvictions(bool fPreSnapshot) {
|
||||
/* After some time, exit the loop early - even if memory limit
|
||||
* hasn't been reached. If we suddenly need to free a lot of
|
||||
* memory, don't want to spend too much time here. */
|
||||
if (g_pserver->m_pstorageFactory == nullptr && elapsedUs(evictionTimer) > eviction_time_limit_us) {
|
||||
if (elapsedUs(evictionTimer) > eviction_time_limit_us) {
|
||||
// We still need to free memory - start eviction timer proc
|
||||
if (!isEvictionProcRunning && serverTL->el != nullptr) {
|
||||
if (!isEvictionProcRunning) {
|
||||
isEvictionProcRunning = 1;
|
||||
aeCreateTimeEvent(serverTL->el, 0,
|
||||
evictionTimeProc, NULL, NULL);
|
||||
@@ -849,38 +721,7 @@ int performEvictions(bool fPreSnapshot) {
|
||||
/* at this point, the memory is OK, or we have reached the time limit */
|
||||
result = (isEvictionProcRunning) ? EVICT_RUNNING : EVICT_OK;
|
||||
|
||||
if (splazy != nullptr && splazy->memory_queued() > 0 && !serverTL->gcEpoch.isReset()) {
|
||||
g_pserver->garbageCollector.enqueue(serverTL->gcEpoch, std::move(splazy));
|
||||
}
|
||||
|
||||
cant_free:
|
||||
if (g_pserver->m_pstorageFactory)
|
||||
{
|
||||
if (mem_reported < g_pserver->maxmemory*1.2) {
|
||||
return EVICT_OK; // Allow us to temporarily go over without OOMing
|
||||
}
|
||||
}
|
||||
|
||||
if (!cserver.delete_on_evict && result == EVICT_FAIL)
|
||||
{
|
||||
for (int idb = 0; idb < cserver.dbnum; ++idb)
|
||||
{
|
||||
redisDb *db = g_pserver->db[idb];
|
||||
if (db->FStorageProvider())
|
||||
{
|
||||
if (db->size() != 0 && db->size(true /*fcachedOnly*/) == 0 && db->keycacheIsEnabled()) {
|
||||
serverLog(LL_WARNING, "Key cache exceeds maxmemory, freeing - performance may be affected increase maxmemory if possible");
|
||||
db->disableKeyCache();
|
||||
} else if (db->size(true /*fCachedOnly*/)) {
|
||||
serverLog(LL_WARNING, "Failed to evict keys, falling back to flushing entire cache. Consider increasing maxmemory-samples.");
|
||||
db->removeAllCachedValues();
|
||||
if (((mem_reported - zmalloc_used_memory()) + mem_freed) >= mem_tofree)
|
||||
result = EVICT_OK;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (result == EVICT_FAIL) {
|
||||
/* At this point, we have run out of evictable items. It's possible
|
||||
* that some items are being freed in the lazyfree thread. Perform a
|
||||
@@ -895,7 +736,6 @@ cant_free:
|
||||
|
||||
latencyEndMonitor(latency);
|
||||
latencyAddSampleIfNeeded("eviction-cycle",latency);
|
||||
cant_free_storage:
|
||||
return result;
|
||||
}
|
||||
|
||||
+46
-56
@@ -33,8 +33,6 @@
|
||||
#include "server.h"
|
||||
#include "cron.h"
|
||||
|
||||
fastlock g_expireLock {"Expire"};
|
||||
|
||||
/* Helper function for the activeExpireCycle() function.
|
||||
* This function will try to expire the key that is stored in the hash table
|
||||
* entry 'de' of the 'expires' hash table of a Redis database.
|
||||
@@ -74,16 +72,16 @@ void activeExpireCycleExpireFullKey(redisDb *db, const char *key) {
|
||||
*----------------------------------------------------------------------------*/
|
||||
|
||||
|
||||
int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &tried) {
|
||||
void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
|
||||
if (!e.FFat())
|
||||
{
|
||||
activeExpireCycleExpireFullKey(db, e.key());
|
||||
++tried;
|
||||
return 1;
|
||||
return;
|
||||
}
|
||||
|
||||
expireEntryFat *pfat = e.pfatentry();
|
||||
robj *val = db->find(e.key());
|
||||
dictEntry *de = dictFind(db->dict, e.key());
|
||||
robj *val = (robj*)dictGetVal(de);
|
||||
int deleted = 0;
|
||||
|
||||
redisObjectStack objKey;
|
||||
@@ -92,7 +90,6 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
|
||||
while (!pfat->FEmpty())
|
||||
{
|
||||
++tried;
|
||||
if (pfat->nextExpireEntry().when > now)
|
||||
break;
|
||||
|
||||
@@ -100,7 +97,7 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
if (pfat->nextExpireEntry().spsubkey == nullptr)
|
||||
{
|
||||
activeExpireCycleExpireFullKey(db, e.key());
|
||||
return ++deleted;
|
||||
return;
|
||||
}
|
||||
|
||||
switch (val->type)
|
||||
@@ -110,7 +107,7 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
deleted++;
|
||||
if (setTypeSize(val) == 0) {
|
||||
activeExpireCycleExpireFullKey(db, e.key());
|
||||
return deleted;
|
||||
return;
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -120,7 +117,7 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
deleted++;
|
||||
if (hashTypeLength(val) == 0) {
|
||||
activeExpireCycleExpireFullKey(db, e.key());
|
||||
return deleted;
|
||||
return;
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -130,7 +127,7 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
deleted++;
|
||||
if (zsetLength(val) == 0) {
|
||||
activeExpireCycleExpireFullKey(db, e.key());
|
||||
return deleted;
|
||||
return;
|
||||
}
|
||||
}
|
||||
break;
|
||||
@@ -145,7 +142,7 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
decrRefCount(val);
|
||||
}, true /*fLock*/, true /*fForceQueue*/);
|
||||
}
|
||||
return deleted;
|
||||
return;
|
||||
|
||||
case OBJ_LIST:
|
||||
default:
|
||||
@@ -158,19 +155,15 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
|
||||
pfat->popfrontExpireEntry();
|
||||
fTtlChanged = true;
|
||||
if ((tried % ACTIVE_EXPIRE_CYCLE_SUBKEY_LOOKUPS_PER_LOOP) == 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (pfat->FEmpty())
|
||||
{
|
||||
removeExpire(db, &objKey);
|
||||
}
|
||||
else if (!pfat->FEmpty() && fTtlChanged)
|
||||
if (!pfat->FEmpty() && fTtlChanged)
|
||||
{
|
||||
// We need to resort the expire entry since it may no longer be in the correct position
|
||||
db->resortExpire(e);
|
||||
auto itr = db->setexpire->find(e.key());
|
||||
expireEntry eT = std::move(e);
|
||||
db->setexpire->erase(itr);
|
||||
db->setexpire->insert(eT);
|
||||
}
|
||||
|
||||
if (deleted)
|
||||
@@ -184,7 +177,10 @@ int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &
|
||||
}
|
||||
}
|
||||
|
||||
return deleted;
|
||||
if (pfat->FEmpty())
|
||||
{
|
||||
removeExpire(db, &objKey);
|
||||
}
|
||||
}
|
||||
|
||||
int parseUnitString(const char *sz)
|
||||
@@ -214,7 +210,7 @@ void expireMemberCore(client *c, robj *key, robj *subkey, long long basetime, lo
|
||||
|
||||
/* No key, return zero. */
|
||||
robj *val = lookupKeyWriteOrReply(c, key, shared.czero);
|
||||
if (val == nullptr) {
|
||||
if (val == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -371,7 +367,7 @@ void activeExpireCycleCore(int type) {
|
||||
long total_expired = 0;
|
||||
|
||||
for (j = 0; j < dbs_per_call && timelimit_exit == 0; j++) {
|
||||
redisDb *db = g_pserver->db[(current_db % cserver.dbnum)];
|
||||
redisDb *db = g_pserver->db+(current_db % cserver.dbnum);
|
||||
|
||||
/* Increment the DB now so we are sure if we run out of time
|
||||
* in the current DB we'll restart from the next. This allows to
|
||||
@@ -383,22 +379,23 @@ void activeExpireCycleCore(int type) {
|
||||
now = mstime();
|
||||
|
||||
/* If there is nothing to expire try next DB ASAP. */
|
||||
if (db->setexpireUnsafe()->empty())
|
||||
if (db->setexpire->empty())
|
||||
{
|
||||
db->avg_ttl = 0;
|
||||
db->last_expire_set = now;
|
||||
continue;
|
||||
}
|
||||
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
size_t expired = 0;
|
||||
size_t tried = 0;
|
||||
long long check = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; // assume a check is roughly 1us. It isn't but good enough
|
||||
db->expireitr = db->setexpireUnsafe()->enumerate(db->expireitr, now, [&](expireEntry &e) __attribute__((always_inline)) {
|
||||
db->expireitr = db->setexpire->enumerate(db->expireitr, now, [&](expireEntry &e) __attribute__((always_inline)) {
|
||||
if (e.when() < now)
|
||||
{
|
||||
expired += activeExpireCycleExpire(db, e, now, tried);
|
||||
activeExpireCycleExpire(db, e, now);
|
||||
++expired;
|
||||
}
|
||||
++tried;
|
||||
|
||||
if ((tried % ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) == 0)
|
||||
{
|
||||
@@ -494,20 +491,20 @@ void expireSlaveKeys(void) {
|
||||
int dbid = 0;
|
||||
while(dbids && dbid < cserver.dbnum) {
|
||||
if ((dbids & 1) != 0) {
|
||||
redisDb *db = g_pserver->db[dbid];
|
||||
redisDb *db = g_pserver->db+dbid;
|
||||
|
||||
// the expire is hashed based on the key pointer, so we need the point in the main db
|
||||
auto itrDB = db->find(keyname);
|
||||
auto itrExpire = db->setexpire()->end();
|
||||
if (itrDB != nullptr)
|
||||
itrExpire = db->setexpireUnsafe()->find(itrDB.key());
|
||||
dictEntry *deMain = dictFind(db->dict, keyname);
|
||||
auto itr = db->setexpire->end();
|
||||
if (deMain != nullptr)
|
||||
itr = db->setexpire->find((sds)dictGetKey(deMain));
|
||||
int expired = 0;
|
||||
|
||||
if (itrExpire != db->setexpire()->end())
|
||||
if (itr != db->setexpire->end())
|
||||
{
|
||||
if (itrExpire->when() < start) {
|
||||
size_t tried = 0;
|
||||
expired = activeExpireCycleExpire(g_pserver->db[dbid],*itrExpire,start,tried);
|
||||
if (itr->when() < start) {
|
||||
activeExpireCycleExpire(g_pserver->db+dbid,*itr,start);
|
||||
expired = 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -515,7 +512,7 @@ void expireSlaveKeys(void) {
|
||||
* corresponding bit in the new bitmap we set as value.
|
||||
* At the end of the loop if the bitmap is zero, it means we
|
||||
* no longer need to keep track of this key. */
|
||||
if (itrExpire != db->setexpire()->end() && !expired) {
|
||||
if (itr != db->setexpire->end() && !expired) {
|
||||
noexpire++;
|
||||
new_dbids |= (uint64_t)1 << dbid;
|
||||
}
|
||||
@@ -683,7 +680,7 @@ void pexpireatCommand(client *c) {
|
||||
|
||||
/* Implements TTL and PTTL */
|
||||
void ttlGenericCommand(client *c, int output_ms) {
|
||||
long long expire = INVALID_EXPIRE, ttl = -1;
|
||||
long long expire = -1, ttl = -1;
|
||||
|
||||
/* If the key does not exist at all, return -2 */
|
||||
if (lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH) == nullptr) {
|
||||
@@ -693,8 +690,7 @@ void ttlGenericCommand(client *c, int output_ms) {
|
||||
|
||||
/* The key exists. Return -1 if it has no expire, or the actual
|
||||
* TTL value otherwise. */
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
expireEntry *pexpire = c->db->getExpire(c->argv[1]);
|
||||
expireEntry *pexpire = getExpire(c->db,c->argv[1]);
|
||||
|
||||
if (c->argc == 2) {
|
||||
// primary expire
|
||||
@@ -718,7 +714,7 @@ void ttlGenericCommand(client *c, int output_ms) {
|
||||
}
|
||||
|
||||
|
||||
if (expire != INVALID_EXPIRE) {
|
||||
if (expire != -1) {
|
||||
ttl = expire-mstime();
|
||||
if (ttl < 0) ttl = 0;
|
||||
}
|
||||
@@ -752,7 +748,7 @@ void persistCommand(client *c) {
|
||||
addReply(c,shared.czero);
|
||||
}
|
||||
} else if (c->argc == 3) {
|
||||
if (c->db->removeSubkeyExpire(c->argv[1], c->argv[2])) {
|
||||
if (removeSubkeyExpire(c->db, c->argv[1], c->argv[2])) {
|
||||
signalModifiedKey(c,c->db,c->argv[1]);
|
||||
notifyKeyspaceEvent(NOTIFY_GENERIC,"persist",c->argv[1],c->db->id);
|
||||
addReply(c,shared.cone);
|
||||
@@ -776,25 +772,19 @@ void touchCommand(client *c) {
|
||||
addReplyLongLong(c,touched);
|
||||
}
|
||||
|
||||
expireEntryFat::expireEntryFat(const expireEntryFat &src)
|
||||
{
|
||||
m_keyPrimary = sdsdupshared(src.m_keyPrimary);
|
||||
m_vecexpireEntries = src.m_vecexpireEntries;
|
||||
m_dictIndex = nullptr;
|
||||
}
|
||||
|
||||
expireEntryFat::~expireEntryFat()
|
||||
{
|
||||
if (m_dictIndex != nullptr)
|
||||
dictRelease(m_dictIndex);
|
||||
}
|
||||
|
||||
expireEntryFat::expireEntryFat(const expireEntryFat &e)
|
||||
: m_keyPrimary(e.m_keyPrimary), m_vecexpireEntries(e.m_vecexpireEntries)
|
||||
{
|
||||
// Note: dictExpires is not copied
|
||||
}
|
||||
|
||||
expireEntryFat::expireEntryFat(expireEntryFat &&e)
|
||||
: m_keyPrimary(std::move(e.m_keyPrimary)), m_vecexpireEntries(std::move(e.m_vecexpireEntries))
|
||||
{
|
||||
m_dictIndex = e.m_dictIndex;
|
||||
e.m_dictIndex = nullptr;
|
||||
}
|
||||
|
||||
void expireEntryFat::createIndex()
|
||||
{
|
||||
serverAssert(m_dictIndex == nullptr);
|
||||
|
||||
+47
-91
@@ -1,12 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
/*
|
||||
* INVALID_EXPIRE is the value we set the expireEntry's m_when value to when the main key is not expired and the value we return when we try to get the expire time of a key or subkey that is not expired
|
||||
* Want this value to be LLONG_MAX however we use the most significant bit of m_when as a flag to see if the expireEntry is Fat or not so we want to ensure that it is unset hence the (& ~((1LL) << (sizeof(long long)*CHAR_BIT - 1)))
|
||||
* Eventually this number might actually end up being a valid expire time, this could cause bugs so at that time it might be a good idea to use a larger data type.
|
||||
*/
|
||||
#define INVALID_EXPIRE (LLONG_MAX & ~((1LL) << (sizeof(long long)*CHAR_BIT - 1)))
|
||||
|
||||
class expireEntryFat
|
||||
{
|
||||
friend class expireEntry;
|
||||
@@ -41,22 +34,20 @@ public:
|
||||
};
|
||||
|
||||
private:
|
||||
sdsimmutablestring m_keyPrimary;
|
||||
sds m_keyPrimary;
|
||||
std::vector<subexpireEntry> m_vecexpireEntries; // Note a NULL for the sds portion means the expire is for the primary key
|
||||
dict *m_dictIndex = nullptr;
|
||||
|
||||
void createIndex();
|
||||
public:
|
||||
expireEntryFat(const sdsimmutablestring &keyPrimary)
|
||||
expireEntryFat(sds keyPrimary)
|
||||
: m_keyPrimary(keyPrimary)
|
||||
{}
|
||||
expireEntryFat(const expireEntryFat &);
|
||||
~expireEntryFat();
|
||||
|
||||
expireEntryFat(const expireEntryFat &e);
|
||||
expireEntryFat(expireEntryFat &&e);
|
||||
|
||||
long long when() const noexcept { return m_vecexpireEntries.front().when; }
|
||||
const char *key() const noexcept { return static_cast<const char*>(m_keyPrimary); }
|
||||
const char *key() const noexcept { return m_keyPrimary; }
|
||||
|
||||
bool operator<(long long when) const noexcept { return this->when() < when; }
|
||||
|
||||
@@ -65,33 +56,28 @@ public:
|
||||
bool FEmpty() const noexcept { return m_vecexpireEntries.empty(); }
|
||||
const subexpireEntry &nextExpireEntry() const noexcept { return m_vecexpireEntries.front(); }
|
||||
void popfrontExpireEntry();
|
||||
const subexpireEntry &operator[](size_t idx) const { return m_vecexpireEntries[idx]; }
|
||||
const subexpireEntry &operator[](size_t idx) { return m_vecexpireEntries[idx]; }
|
||||
size_t size() const noexcept { return m_vecexpireEntries.size(); }
|
||||
};
|
||||
|
||||
class expireEntry {
|
||||
struct
|
||||
union
|
||||
{
|
||||
sdsimmutablestring m_key;
|
||||
expireEntryFat *m_pfatentry = nullptr;
|
||||
sds m_key;
|
||||
expireEntryFat *m_pfatentry;
|
||||
} u;
|
||||
long long m_when; // bit wise and with FFatMask means this is a fat entry and we should use the pointer
|
||||
|
||||
/* Mask to check if an entry is Fat, most significant bit of m_when being set means it is Fat otherwise it is not */
|
||||
long long FFatMask() const noexcept {
|
||||
return (1LL) << (sizeof(long long)*CHAR_BIT - 1);
|
||||
}
|
||||
long long m_when; // LLONG_MIN means this is a fat entry and we should use the pointer
|
||||
|
||||
expireEntry() = default;
|
||||
public:
|
||||
class iter
|
||||
{
|
||||
friend class expireEntry;
|
||||
const expireEntry *m_pentry = nullptr;
|
||||
expireEntry *m_pentry = nullptr;
|
||||
size_t m_idx = 0;
|
||||
|
||||
public:
|
||||
iter(const expireEntry *pentry, size_t idx)
|
||||
iter(expireEntry *pentry, size_t idx)
|
||||
: m_pentry(pentry), m_idx(idx)
|
||||
{}
|
||||
|
||||
@@ -122,42 +108,29 @@ public:
|
||||
{
|
||||
if (subkey != nullptr)
|
||||
{
|
||||
m_when = FFatMask() | INVALID_EXPIRE;
|
||||
u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(sdsimmutablestring(sdsdupshared(key)));
|
||||
m_when = LLONG_MIN;
|
||||
u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(key);
|
||||
u.m_pfatentry->expireSubKey(subkey, when);
|
||||
}
|
||||
else
|
||||
{
|
||||
u.m_key = sdsimmutablestring(sdsdupshared(key));
|
||||
u.m_key = key;
|
||||
m_when = when;
|
||||
}
|
||||
}
|
||||
|
||||
expireEntry(const expireEntry &e)
|
||||
{
|
||||
*this = e;
|
||||
}
|
||||
expireEntry(expireEntry &&e)
|
||||
{
|
||||
u.m_key = std::move(e.u.m_key);
|
||||
u.m_pfatentry = std::move(e.u.m_pfatentry);
|
||||
m_when = e.m_when;
|
||||
e.m_when = 0;
|
||||
e.u.m_pfatentry = nullptr;
|
||||
}
|
||||
|
||||
expireEntry(expireEntryFat *pfatentry)
|
||||
{
|
||||
u.m_pfatentry = pfatentry;
|
||||
m_when = FFatMask() | INVALID_EXPIRE;
|
||||
for (auto itr : *this)
|
||||
{
|
||||
if (itr.subkey() == nullptr)
|
||||
{
|
||||
m_when = FFatMask() | itr.when();
|
||||
break;
|
||||
}
|
||||
}
|
||||
m_when = LLONG_MIN;
|
||||
}
|
||||
|
||||
expireEntry(expireEntry &&e)
|
||||
{
|
||||
u.m_key = e.u.m_key;
|
||||
m_when = e.m_when;
|
||||
e.u.m_key = (char*)key(); // we do this so it can still be found in the set
|
||||
e.m_when = 0;
|
||||
}
|
||||
|
||||
// Duplicate the expire, note this is intended to be passed directly to setExpire
|
||||
@@ -178,31 +151,21 @@ public:
|
||||
delete u.m_pfatentry;
|
||||
}
|
||||
|
||||
expireEntry &operator=(const expireEntry &e)
|
||||
{
|
||||
u.m_key = e.u.m_key;
|
||||
m_when = e.m_when;
|
||||
if (e.FFat())
|
||||
u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(*e.u.m_pfatentry);
|
||||
return *this;
|
||||
}
|
||||
|
||||
void setKeyUnsafe(sds key)
|
||||
{
|
||||
if (FFat())
|
||||
u.m_pfatentry->m_keyPrimary = sdsimmutablestring(sdsdupshared(key));
|
||||
u.m_pfatentry->m_keyPrimary = key;
|
||||
else
|
||||
u.m_key = sdsimmutablestring(sdsdupshared(key));
|
||||
u.m_key = key;
|
||||
}
|
||||
|
||||
inline bool FFat() const noexcept { return m_when & FFatMask(); }
|
||||
inline bool FFat() const noexcept { return m_when == LLONG_MIN; }
|
||||
expireEntryFat *pfatentry() { assert(FFat()); return u.m_pfatentry; }
|
||||
const expireEntryFat *pfatentry() const { assert(FFat()); return u.m_pfatentry; }
|
||||
|
||||
|
||||
bool operator==(const sdsview &key) const noexcept
|
||||
bool operator==(const char *key) const noexcept
|
||||
{
|
||||
return key == this->key();
|
||||
return this->key() == key;
|
||||
}
|
||||
|
||||
bool operator<(const expireEntry &e) const noexcept
|
||||
@@ -218,13 +181,13 @@ public:
|
||||
{
|
||||
if (FFat())
|
||||
return u.m_pfatentry->key();
|
||||
return static_cast<const char*>(u.m_key);
|
||||
return u.m_key;
|
||||
}
|
||||
long long when() const noexcept
|
||||
{
|
||||
if (FFat())
|
||||
return u.m_pfatentry->when();
|
||||
return FGetPrimaryExpire();
|
||||
return m_when;
|
||||
}
|
||||
|
||||
void update(const char *subkey, long long when)
|
||||
@@ -240,20 +203,18 @@ public:
|
||||
{
|
||||
// we have to upgrade to a fat entry
|
||||
long long whenT = m_when;
|
||||
sdsimmutablestring keyPrimary = u.m_key;
|
||||
m_when |= FFatMask();
|
||||
sds keyPrimary = u.m_key;
|
||||
m_when = LLONG_MIN;
|
||||
u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(keyPrimary);
|
||||
u.m_pfatentry->expireSubKey(nullptr, whenT);
|
||||
// at this point we're fat so fall through
|
||||
}
|
||||
}
|
||||
if (subkey == nullptr)
|
||||
m_when = when | FFatMask();
|
||||
u.m_pfatentry->expireSubKey(subkey, when);
|
||||
}
|
||||
|
||||
iter begin() const { return iter(this, 0); }
|
||||
iter end() const
|
||||
iter begin() { return iter(this, 0); }
|
||||
iter end()
|
||||
{
|
||||
if (FFat())
|
||||
return iter(this, u.m_pfatentry->size());
|
||||
@@ -268,26 +229,21 @@ public:
|
||||
pfatentry()->m_vecexpireEntries.begin() + itr.m_idx);
|
||||
}
|
||||
|
||||
size_t size() const
|
||||
bool FGetPrimaryExpire(long long *pwhen)
|
||||
{
|
||||
if (FFat())
|
||||
return u.m_pfatentry->size();
|
||||
return 1;
|
||||
*pwhen = -1;
|
||||
for (auto itr : *this)
|
||||
{
|
||||
if (itr.subkey() == nullptr)
|
||||
{
|
||||
*pwhen = itr.when();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
long long FGetPrimaryExpire() const noexcept
|
||||
{
|
||||
return m_when & (~FFatMask());
|
||||
}
|
||||
|
||||
bool FGetPrimaryExpire(long long *pwhen) const noexcept
|
||||
{
|
||||
*pwhen = FGetPrimaryExpire();
|
||||
return *pwhen != INVALID_EXPIRE;
|
||||
}
|
||||
|
||||
explicit operator sdsview() const noexcept { return key(); }
|
||||
explicit operator const char*() const noexcept { return key(); }
|
||||
explicit operator long long() const noexcept { return when(); }
|
||||
};
|
||||
typedef semiorderedset<expireEntry, sdsview, true /*expireEntry can be memmoved*/> expireset;
|
||||
extern fastlock g_expireLock;
|
||||
typedef semiorderedset<expireEntry, const char *, true /*expireEntry can be memmoved*/> expireset;
|
||||
+1
-1
@@ -437,7 +437,7 @@ extern "C" void fastlock_unlock(struct fastlock *lock)
|
||||
serverAssert(pidT >= 0); // unlock after free
|
||||
int t = -1;
|
||||
__atomic_store(&lock->m_pidOwner, &t, __ATOMIC_RELEASE);
|
||||
std::atomic_thread_fence(std::memory_order_acq_rel);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
ANNOTATE_RWLOCK_RELEASED(lock, true);
|
||||
uint16_t activeNew = __atomic_add_fetch(&lock->m_ticket.m_active, 1, __ATOMIC_RELEASE); // on x86 the atomic is not required here, but ASM handles that case
|
||||
#ifdef __linux__
|
||||
|
||||
@@ -190,11 +190,10 @@ fastlock_unlock:
|
||||
mov dword ptr [rdi], -1 # pidOwner = -1 (we don't own it anymore)
|
||||
mov esi, [rdi+64] # get current active (this one)
|
||||
inc esi # bump it to the next thread
|
||||
sfence # ensure whatever was written in the lock is visible
|
||||
mov word ptr [rdi+64], si # give up our ticket (note: lock is not required here because the spinlock itself guards this variable)
|
||||
mfence # sync other threads
|
||||
# At this point the lock is removed, however we must wake up any pending futexs
|
||||
mov rdx, [rdi+64] # load the futex mask, note we intentionally also read the ticket we just wrote to ensure this is ordered with the above mov
|
||||
shr rdx, 32 # isolate the mask
|
||||
mov edx, [rdi+64+4] # load the futex mask
|
||||
bt edx, esi # is the next thread waiting on a futex?
|
||||
jc unlock_futex # unlock the futex if necessary
|
||||
ret # if not we're done.
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
#pragma once
|
||||
#include <vector>
|
||||
#include <assert.h>
|
||||
#include <unordered_set>
|
||||
#include <list>
|
||||
|
||||
struct ICollectable
|
||||
{
|
||||
virtual ~ICollectable() {}
|
||||
bool FWillFreeChildDebug() { return false; }
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
class GarbageCollector
|
||||
{
|
||||
struct EpochHolder
|
||||
{
|
||||
uint64_t tstamp;
|
||||
std::unique_ptr<std::vector<std::unique_ptr<T>>> m_spvecObjs;
|
||||
|
||||
EpochHolder() {
|
||||
m_spvecObjs = std::make_unique<std::vector<std::unique_ptr<T>>>();
|
||||
}
|
||||
|
||||
// Support move operators
|
||||
EpochHolder(EpochHolder &&other) = default;
|
||||
EpochHolder &operator=(EpochHolder &&) = default;
|
||||
|
||||
|
||||
|
||||
bool operator<(uint64_t tstamp) const
|
||||
{
|
||||
return this->tstamp < tstamp;
|
||||
}
|
||||
|
||||
bool operator==(uint64_t tstamp) const
|
||||
{
|
||||
return this->tstamp == tstamp;
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
~GarbageCollector() {
|
||||
// Silence TSAN errors
|
||||
m_lock.lock();
|
||||
}
|
||||
|
||||
uint64_t startEpoch()
|
||||
{
|
||||
std::unique_lock<fastlock> lock(m_lock);
|
||||
++m_epochNext;
|
||||
m_setepochOutstanding.insert(m_epochNext);
|
||||
return m_epochNext;
|
||||
}
|
||||
|
||||
void shutdown()
|
||||
{
|
||||
std::unique_lock<fastlock> lock(m_lock);
|
||||
m_listepochs.clear();
|
||||
m_setepochOutstanding.clear();
|
||||
}
|
||||
|
||||
bool empty() const
|
||||
{
|
||||
std::unique_lock<fastlock> lock(m_lock);
|
||||
return m_listepochs.empty();
|
||||
}
|
||||
|
||||
void endEpoch(uint64_t epoch, bool fNoFree = false)
|
||||
{
|
||||
std::unique_lock<fastlock> lock(m_lock);
|
||||
serverAssert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end());
|
||||
bool fMinElement = *std::min_element(m_setepochOutstanding.begin(), m_setepochOutstanding.end());
|
||||
m_setepochOutstanding.erase(epoch);
|
||||
if (fNoFree)
|
||||
return;
|
||||
std::list<EpochHolder> listclean;
|
||||
|
||||
// No outstanding epochs?
|
||||
if (m_setepochOutstanding.empty())
|
||||
{
|
||||
listclean = std::move(m_listepochs); // Everything goes!
|
||||
}
|
||||
else
|
||||
{
|
||||
uint64_t minepoch = *std::min_element(m_setepochOutstanding.begin(), m_setepochOutstanding.end());
|
||||
if (minepoch == 0)
|
||||
return; // No available epochs to free
|
||||
|
||||
// Clean any epochs available (after the lock)
|
||||
for (auto itr = m_listepochs.begin(); itr != m_listepochs.end(); /* itr incremented in loop*/)
|
||||
{
|
||||
auto &e = *itr;
|
||||
auto itrNext = itr;
|
||||
++itrNext;
|
||||
if (e < minepoch)
|
||||
{
|
||||
listclean.emplace_back(std::move(e));
|
||||
m_listepochs.erase(itr);
|
||||
}
|
||||
itr = itrNext;
|
||||
}
|
||||
|
||||
assert(listclean.empty() || fMinElement);
|
||||
}
|
||||
|
||||
lock.unlock(); // don't hold it for the potentially long delete of vecclean
|
||||
}
|
||||
|
||||
void enqueue(uint64_t epoch, std::unique_ptr<T> &&sp)
|
||||
{
|
||||
std::unique_lock<fastlock> lock(m_lock);
|
||||
serverAssert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end());
|
||||
serverAssert(sp->FWillFreeChildDebug() == false);
|
||||
|
||||
auto itr = std::find(m_listepochs.begin(), m_listepochs.end(), m_epochNext+1);
|
||||
if (itr == m_listepochs.end())
|
||||
{
|
||||
EpochHolder e;
|
||||
e.tstamp = m_epochNext+1;
|
||||
e.m_spvecObjs->push_back(std::move(sp));
|
||||
m_listepochs.emplace_back(std::move(e));
|
||||
}
|
||||
else
|
||||
{
|
||||
itr->m_spvecObjs->push_back(std::move(sp));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
mutable fastlock m_lock { "Garbage Collector"};
|
||||
|
||||
std::list<EpochHolder> m_listepochs;
|
||||
std::unordered_set<uint64_t> m_setepochOutstanding;
|
||||
uint64_t m_epochNext = 0;
|
||||
};
|
||||
+8
-37
@@ -509,7 +509,7 @@ void geoaddCommand(client *c) {
|
||||
* [COUNT count [ANY]] [STORE key] [STOREDIST key]
|
||||
* GEORADIUSBYMEMBER key member radius unit ... options ...
|
||||
* GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
|
||||
* [BYBOX width height unit] [WITHCOORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC]
|
||||
* [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC]
|
||||
* GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
|
||||
* [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] [STOREDIST]
|
||||
* */
|
||||
@@ -518,24 +518,21 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */
|
||||
|
||||
/* Look up the requested zset */
|
||||
robj_roptr zobj = lookupKeyRead(c->db, c->argv[srcKeyIndex]);
|
||||
if (checkType(c, zobj, OBJ_ZSET)) return;
|
||||
robj_roptr zobj = nullptr;
|
||||
if ((zobj = lookupKeyReadOrReply(c, c->argv[srcKeyIndex], shared.emptyarray)) == nullptr ||
|
||||
checkType(c, zobj, OBJ_ZSET)) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Find long/lat to use for radius or box search based on inquiry type */
|
||||
int base_args;
|
||||
GeoShape shape = {0};
|
||||
if (flags & RADIUS_COORDS) {
|
||||
/* GEORADIUS or GEORADIUS_RO */
|
||||
base_args = 6;
|
||||
shape.type = CIRCULAR_TYPE;
|
||||
if (extractLongLatOrReply(c, c->argv + 2, shape.xy) == C_ERR) return;
|
||||
if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
|
||||
} else if ((flags & RADIUS_MEMBER) && !zobj) {
|
||||
/* We don't have a source key, but we need to proceed with argument
|
||||
* parsing, so we know which reply to use depending on the STORE flag. */
|
||||
base_args = 5;
|
||||
} else if (flags & RADIUS_MEMBER) {
|
||||
/* GEORADIUSBYMEMBER or GEORADIUSBYMEMBER_RO */
|
||||
base_args = 5;
|
||||
shape.type = CIRCULAR_TYPE;
|
||||
robj *member = c->argv[2];
|
||||
@@ -545,7 +542,6 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
}
|
||||
if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
|
||||
} else if (flags & GEOSEARCH) {
|
||||
/* GEOSEARCH or GEOSEARCHSTORE */
|
||||
base_args = 2;
|
||||
if (flags & GEOSEARCHSTORE) {
|
||||
base_args = 3;
|
||||
@@ -612,13 +608,6 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
flags & GEOSEARCH &&
|
||||
!fromloc)
|
||||
{
|
||||
/* No source key, proceed with argument parsing and return an error when done. */
|
||||
if (!zobj) {
|
||||
frommember = 1;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (longLatFromMember(zobj, c->argv[base_args+i+1], shape.xy) == C_ERR) {
|
||||
addReplyError(c, "could not decode requested zset member");
|
||||
return;
|
||||
@@ -687,23 +676,6 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Return ASAP when src key does not exist. */
|
||||
if (!zobj) {
|
||||
if (storekey) {
|
||||
/* store key is not NULL, try to delete it and return 0. */
|
||||
if (dbDelete(c->db, storekey)) {
|
||||
signalModifiedKey(c, c->db, storekey);
|
||||
notifyKeyspaceEvent(NOTIFY_GENERIC, "del", storekey, c->db->id);
|
||||
g_pserver->dirty++;
|
||||
}
|
||||
addReply(c, shared.czero);
|
||||
} else {
|
||||
/* Otherwise we return an empty array. */
|
||||
addReply(c, shared.emptyarray);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/* COUNT without ordering does not make much sense (we need to
|
||||
* sort in order to return the closest N entries),
|
||||
* force ASC ordering if COUNT was specified but no sorting was
|
||||
@@ -798,7 +770,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
robj *zobj;
|
||||
zset *zs;
|
||||
int i;
|
||||
size_t maxelelen = 0, totelelen = 0;
|
||||
size_t maxelelen = 0;
|
||||
|
||||
if (returned_items) {
|
||||
zobj = createZsetObject();
|
||||
@@ -813,14 +785,13 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
|
||||
size_t elelen = sdslen(gp->member);
|
||||
|
||||
if (maxelelen < elelen) maxelelen = elelen;
|
||||
totelelen += elelen;
|
||||
znode = zslInsert(zs->zsl,score,gp->member);
|
||||
serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);
|
||||
gp->member = NULL;
|
||||
}
|
||||
|
||||
if (returned_items) {
|
||||
zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen);
|
||||
zsetConvertToZiplistIfNeeded(zobj,maxelelen);
|
||||
setKey(c,c->db,storekey,zobj);
|
||||
decrRefCount(zobj);
|
||||
notifyKeyspaceEvent(NOTIFY_ZSET,flags & GEOSEARCH ? "geosearchstore" : "georadiusstore",storekey,
|
||||
|
||||
+1
-1
@@ -1071,7 +1071,7 @@ struct commandHelp {
|
||||
1,
|
||||
"2.2.0" },
|
||||
{ "SHUTDOWN",
|
||||
"[NOSAVE|SAVE|SOFT]",
|
||||
"[NOSAVE|SAVE]",
|
||||
"Synchronously save the dataset to disk and then shut down the server",
|
||||
9,
|
||||
"1.0.0" },
|
||||
|
||||
+1
-2
@@ -104,8 +104,7 @@ intset *intsetNew(void) {
|
||||
|
||||
/* Resize the intset */
|
||||
static intset *intsetResize(intset *is, uint32_t len) {
|
||||
uint64_t size = (uint64_t)len*intrev32ifbe(is->encoding);
|
||||
assert(size <= SIZE_MAX - sizeof(intset));
|
||||
uint32_t size = len*intrev32ifbe(is->encoding);
|
||||
is = zrealloc(is,sizeof(intset)+size, MALLOC_SHARED);
|
||||
return is;
|
||||
}
|
||||
|
||||
@@ -211,11 +211,6 @@ extern "C" void _serverAssert(const char *estr, const char *file, int line) {
|
||||
*((char*)-1) = 'x';
|
||||
}
|
||||
|
||||
/* asyncFreeDictTable is needed by dict */
|
||||
extern "C" void asyncFreeDictTable(struct dictEntry **de) {
|
||||
zfree(de);
|
||||
}
|
||||
|
||||
static redisContext *getRedisContext(const char *ip, int port,
|
||||
const char *hostsocket)
|
||||
{
|
||||
@@ -904,7 +899,7 @@ int main(int argc, const char **argv) {
|
||||
|
||||
while (self_threads < config.max_threads) {
|
||||
for (int i = 0; i < config.numclients; i++) {
|
||||
snprintf(command, 63, "SET %d %s\r\n", self_threads * config.numclients + i, set_value);
|
||||
sprintf(command, "SET %d %s\r\n", self_threads * config.numclients + i, set_value);
|
||||
createClient(command, strlen(command), NULL,self_threads);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
#include "server.h"
|
||||
|
||||
namespace keydbutils
|
||||
{
|
||||
template<>
|
||||
size_t hash(const sdsview& t)
|
||||
{
|
||||
return (size_t)dictGenHashFunction(static_cast<const char*>(t), t.size());
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -230,7 +230,7 @@ void analyzeLatencyForEvent(char *event, struct latencyStats *ls) {
|
||||
if (ls->samples) ls->mad = sum / ls->samples;
|
||||
}
|
||||
|
||||
/* Create a human readable report of latency events for this KeyDB instance. */
|
||||
/* Create a human readable report of latency events for this Redis instance. */
|
||||
sds createLatencyReport(void) {
|
||||
sds report = sdsempty();
|
||||
int advise_better_vm = 0; /* Better virtual machines. */
|
||||
@@ -256,7 +256,7 @@ sds createLatencyReport(void) {
|
||||
if (dictSize(g_pserver->latency_events) == 0 &&
|
||||
g_pserver->latency_monitor_threshold == 0)
|
||||
{
|
||||
report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this KeyDB instance. You may use \"CONFIG SET latency-monitor-threshold <milliseconds>.\" in order to enable it.\n");
|
||||
report = sdscat(report,"I'm sorry, Dave, I can't do that. Latency monitoring is disabled in this KeyDB instance. You may use \"CONFIG SET latency-monitor-threshold <milliseconds>.\" in order to enable it. If we weren't in a deep space mission I'd suggest to take a look at https://redis.io/topics/latency-monitor.\n");
|
||||
return report;
|
||||
}
|
||||
|
||||
@@ -398,7 +398,7 @@ sds createLatencyReport(void) {
|
||||
dictReleaseIterator(di);
|
||||
|
||||
/* Add non event based advices. */
|
||||
if (g_pserver->aof_enabled && THPGetAnonHugePagesSize() > 0) {
|
||||
if (THPGetAnonHugePagesSize() > 0) {
|
||||
advise_disable_thp = 1;
|
||||
advices++;
|
||||
}
|
||||
|
||||
+15
-32
@@ -109,7 +109,7 @@ size_t lazyfreeGetFreeEffort(robj *key, robj *obj) {
|
||||
/* Every consumer group is an allocation and so are the entries in its
|
||||
* PEL. We use size of the first group's PEL as an estimate for all
|
||||
* others. */
|
||||
if (s->cgroups && raxSize(s->cgroups)) {
|
||||
if (s->cgroups) {
|
||||
raxIterator ri;
|
||||
streamCG *cg;
|
||||
raxStart(&ri,s->cgroups);
|
||||
@@ -143,28 +143,19 @@ size_t lazyfreeGetFreeEffort(robj *key, robj *obj) {
|
||||
* a lazy free list instead of being freed synchronously. The lazy free list
|
||||
* will be reclaimed in a different bio.c thread. */
|
||||
#define LAZYFREE_THRESHOLD 64
|
||||
bool redisDbPersistentData::asyncDelete(robj *key) {
|
||||
int dbAsyncDelete(redisDb *db, robj *key) {
|
||||
/* If the value is composed of a few allocations, to free in a lazy way
|
||||
* is actually just slower... So under a certain limit we just free
|
||||
* the object synchronously. */
|
||||
|
||||
if (m_spstorage != nullptr)
|
||||
return syncDelete(key); // async delte never makes sense with a storage provider
|
||||
|
||||
dictEntry *de = dictUnlink(m_pdict,ptrFromObj(key));
|
||||
dictEntry *de = dictUnlink(db->dict,ptrFromObj(key));
|
||||
if (de) {
|
||||
if (m_pdbSnapshot != nullptr && m_pdbSnapshot->find_cached_threadsafe(szFromObj(key)) != nullptr)
|
||||
{
|
||||
uint64_t hash = dictGetHash(m_pdict, szFromObj(key));
|
||||
dictAdd(m_pdictTombstone, sdsdup((sds)dictGetKey(de)), (void*)hash);
|
||||
}
|
||||
|
||||
robj *val = (robj*)dictGetVal(de);
|
||||
|
||||
if (val->FExpires())
|
||||
{
|
||||
/* Deleting an entry from the expires dict will not free the sds of
|
||||
* the key, because it is shared with the main dictionary. */
|
||||
removeExpire(key,dict_iter(m_pdict, de));
|
||||
removeExpireCore(db,key,de);
|
||||
}
|
||||
|
||||
/* Tells the module that the key has been unlinked from the database. */
|
||||
@@ -183,25 +174,21 @@ bool redisDbPersistentData::asyncDelete(robj *key) {
|
||||
if (free_effort > LAZYFREE_THRESHOLD && val->getrefcount(std::memory_order_relaxed) == 1) {
|
||||
atomicIncr(lazyfree_objects,1);
|
||||
bioCreateLazyFreeJob(lazyfreeFreeObject,1, val);
|
||||
dictSetVal(m_pdict,de,NULL);
|
||||
dictSetVal(db->dict,de,NULL);
|
||||
}
|
||||
}
|
||||
|
||||
/* Release the key-val pair, or just the key if we set the val
|
||||
* field to NULL in order to lazy free it later. */
|
||||
if (de) {
|
||||
dictFreeUnlinkedEntry(m_pdict,de);
|
||||
dictFreeUnlinkedEntry(db->dict,de);
|
||||
if (g_pserver->cluster_enabled) slotToKeyDel(szFromObj(key));
|
||||
return true;
|
||||
return 1;
|
||||
} else {
|
||||
return false;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
int dbAsyncDelete(redisDb *db, robj *key) {
|
||||
return db->asyncDelete(key);
|
||||
}
|
||||
|
||||
/* Free an object, if the object is huge enough, free it in async way. */
|
||||
void freeObjAsync(robj *key, robj *obj) {
|
||||
size_t free_effort = lazyfreeGetFreeEffort(key,obj);
|
||||
@@ -216,16 +203,12 @@ void freeObjAsync(robj *key, robj *obj) {
|
||||
/* Empty a Redis DB asynchronously. What the function does actually is to
|
||||
* create a new empty set of hash tables and scheduling the old ones for
|
||||
* lazy freeing. */
|
||||
void redisDbPersistentData::emptyDbAsync() {
|
||||
std::unique_lock<fastlock> ul(g_expireLock);
|
||||
dict *oldht1 = m_pdict;
|
||||
auto *set = m_setexpire;
|
||||
m_setexpire = new (MALLOC_LOCAL) expireset();
|
||||
m_pdict = dictCreate(&dbDictType,this);
|
||||
if (m_spstorage != nullptr)
|
||||
m_spstorage->clearAsync();
|
||||
if (m_fTrackingChanges)
|
||||
m_fAllChanged = true;
|
||||
void emptyDbAsync(redisDb *db) {
|
||||
dict *oldht1 = db->dict;
|
||||
auto *set = db->setexpire;
|
||||
db->setexpire = new (MALLOC_LOCAL) expireset();
|
||||
db->expireitr = db->setexpire->end();
|
||||
db->dict = dictCreate(&dbDictType,NULL);
|
||||
atomicIncr(lazyfree_objects,dictSize(oldht1));
|
||||
bioCreateLazyFreeJob(lazyfreeFreeDatabase,2,oldht1,set);
|
||||
}
|
||||
|
||||
+8
-27
@@ -131,8 +131,6 @@
|
||||
assert((p) >= (lp)+LP_HDR_SIZE && (p)+(len) < (lp)+lpGetTotalBytes((lp))); \
|
||||
} while (0)
|
||||
|
||||
static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p);
|
||||
|
||||
/* Convert a string into a signed 64 bit integer.
|
||||
* The function returns 1 if the string could be parsed into a (non-overflowing)
|
||||
* signed 64 bit int, 0 otherwise. The 'value' will be set to the parsed value
|
||||
@@ -315,7 +313,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui
|
||||
} else {
|
||||
if (size < 64) *enclen = 1+size;
|
||||
else if (size < 4096) *enclen = 2+size;
|
||||
else *enclen = 5+(uint64_t)size;
|
||||
else *enclen = 5+size;
|
||||
return LP_ENCODING_STRING;
|
||||
}
|
||||
}
|
||||
@@ -455,8 +453,8 @@ unsigned char *lpSkip(unsigned char *p) {
|
||||
unsigned char *lpNext(unsigned char *lp, unsigned char *p) {
|
||||
assert(p);
|
||||
p = lpSkip(p);
|
||||
ASSERT_INTEGRITY(lp, p);
|
||||
if (p[0] == LP_EOF) return NULL;
|
||||
lpAssertValidEntry(lp, lpBytes(lp), p);
|
||||
return p;
|
||||
}
|
||||
|
||||
@@ -470,17 +468,16 @@ unsigned char *lpPrev(unsigned char *lp, unsigned char *p) {
|
||||
uint64_t prevlen = lpDecodeBacklen(p);
|
||||
prevlen += lpEncodeBacklen(NULL,prevlen);
|
||||
p -= prevlen-1; /* Seek the first byte of the previous entry. */
|
||||
lpAssertValidEntry(lp, lpBytes(lp), p);
|
||||
ASSERT_INTEGRITY(lp, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Return a pointer to the first element of the listpack, or NULL if the
|
||||
* listpack has no elements. */
|
||||
unsigned char *lpFirst(unsigned char *lp) {
|
||||
unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */
|
||||
if (p[0] == LP_EOF) return NULL;
|
||||
lpAssertValidEntry(lp, lpBytes(lp), p);
|
||||
return p;
|
||||
lp += LP_HDR_SIZE; /* Skip the header. */
|
||||
if (lp[0] == LP_EOF) return NULL;
|
||||
return lp;
|
||||
}
|
||||
|
||||
/* Return a pointer to the last element of the listpack, or NULL if the
|
||||
@@ -864,13 +861,6 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
|
||||
}
|
||||
}
|
||||
|
||||
/* Same as lpFirst but without validation assert, to be used right before lpValidateNext. */
|
||||
unsigned char *lpValidateFirst(unsigned char *lp) {
|
||||
unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */
|
||||
if (p[0] == LP_EOF) return NULL;
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Validate the integrity of a single listpack entry and move to the next one.
|
||||
* The input argument 'pp' is a reference to the current record and is advanced on exit.
|
||||
* Returns 1 if valid, 0 if invalid. */
|
||||
@@ -882,10 +872,6 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
|
||||
if (!p)
|
||||
return 0;
|
||||
|
||||
/* Before accessing p, make sure it's valid. */
|
||||
if (OUT_OF_RANGE(p))
|
||||
return 0;
|
||||
|
||||
if (*p == LP_EOF) {
|
||||
*pp = NULL;
|
||||
return 1;
|
||||
@@ -922,11 +908,6 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
|
||||
#undef OUT_OF_RANGE
|
||||
}
|
||||
|
||||
/* Validate that the entry doesn't reach outside the listpack allocation. */
|
||||
static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p) {
|
||||
assert(lpValidateNext(lp, &p, lpbytes));
|
||||
}
|
||||
|
||||
/* Validate the integrity of the data structure.
|
||||
* when `deep` is 0, only the integrity of the header is validated.
|
||||
* when `deep` is 1, we scan all the entries one by one. */
|
||||
@@ -949,8 +930,8 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
|
||||
|
||||
/* Validate the invividual entries. */
|
||||
uint32_t count = 0;
|
||||
unsigned char *p = lp + LP_HDR_SIZE;
|
||||
while(p && p[0] != LP_EOF) {
|
||||
unsigned char *p = lpFirst(lp);
|
||||
while(p) {
|
||||
if (!lpValidateNext(lp, &p, bytes))
|
||||
return 0;
|
||||
count++;
|
||||
|
||||
Loaded 100 of 232 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user