From 96d9d13836a1aca8a7712b6e11f40124f7f5264d Mon Sep 17 00:00:00 2001 From: wubin1989 <328454505@qq.com> Date: Sun, 17 Sep 2023 15:03:45 +0800 Subject: [PATCH] ... --- go.mod | 20 +- go.sum | 20 +- toolkit/cache/.prettierrc.yml | 4 + toolkit/cache/CHANGELOG.md | 9 + toolkit/cache/LICENSE | 25 ++ toolkit/cache/Makefile | 6 + toolkit/cache/README.md | 122 ++++++++ toolkit/cache/bench_test.go | 58 ++++ toolkit/cache/cache.go | 404 ++++++++++++++++++++++++++ toolkit/cache/cache_test.go | 426 ++++++++++++++++++++++++++++ toolkit/cache/example_cache_test.go | 84 ++++++ toolkit/cache/local.go | 81 ++++++ toolkit/cache/local_test.go | 56 ++++ toolkit/gormgen/tests/go.mod | 9 +- toolkit/sqlext/logger/logger.go | 29 +- toolkit/sqlext/wrapper/wrapper.go | 3 +- 16 files changed, 1313 insertions(+), 43 deletions(-) create mode 100644 toolkit/cache/.prettierrc.yml create mode 100644 toolkit/cache/CHANGELOG.md create mode 100644 toolkit/cache/LICENSE create mode 100644 toolkit/cache/Makefile create mode 100644 toolkit/cache/README.md create mode 100644 toolkit/cache/bench_test.go create mode 100644 toolkit/cache/cache.go create mode 100644 toolkit/cache/cache_test.go create mode 100644 toolkit/cache/example_cache_test.go create mode 100644 toolkit/cache/local.go create mode 100644 toolkit/cache/local_test.go diff --git a/go.mod b/go.mod index 4db6a4e4..99d93811 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,12 @@ require ( gorm.io/plugin/dbresolver v1.3.0 ) +require ( + github.com/google/go-cmp v0.5.9 // indirect + github.com/nxadm/tail v1.4.8 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect +) + require ( github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795 // indirect github.com/ClickHouse/ch-go v0.48.0 // indirect @@ -68,7 +74,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cloudflare/circl v1.1.0 // indirect github.com/containerd/cgroups v1.0.3 // indirect @@ -111,6 +117,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.16.7 github.com/klauspost/pgzip v1.2.6 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.1 // indirect @@ -124,6 +131,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/nwaples/rardecode v1.1.3 // indirect + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.25.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/opencontainers/runc v1.1.5 // indirect @@ -136,6 +145,7 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect + github.com/redis/go-redis/v9 v9.1.0 github.com/rivo/uniseg v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/skeema/knownhosts v1.1.0 // indirect @@ -149,8 +159,8 @@ require ( github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/ulikunitz/xz v0.5.11 // indirect - github.com/vmihailenco/go-tinylfu v0.2.2 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect + github.com/vmihailenco/go-tinylfu v0.2.2 + github.com/vmihailenco/msgpack/v5 v5.3.4 github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect @@ -165,7 +175,7 @@ require ( go.uber.org/zap v1.23.0 // indirect golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/sync v0.3.0 golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.10.0 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect @@ -190,7 +200,6 @@ require ( github.com/go-playground/form/v4 v4.2.0 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.11.2 - github.com/go-redis/cache/v8 v8.4.4 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 github.com/google/btree v1.1.2 @@ -205,7 +214,6 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/logutils v1.0.0 github.com/jackc/pgx/v5 v5.3.1 // indirect - github.com/klauspost/compress v1.16.7 github.com/lib/pq v1.10.2 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 github.com/mattn/go-colorable v0.1.13 diff --git a/go.sum b/go.sum index 06f6cf11..0358d022 100644 --- a/go.sum +++ b/go.sum @@ -147,6 +147,8 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= +github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= +github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -161,8 +163,9 @@ github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= @@ -399,6 +402,7 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= @@ -424,11 +428,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= -github.com/go-redis/cache/v8 v8.4.4 h1:Rm0wZ55X22BA2JMqVtRQNHYyzDd0I5f+Ec/C9Xx3mXY= -github.com/go-redis/cache/v8 v8.4.4/go.mod h1:JM6CkupsPvAu/LYEVGQy6UB4WDAzQSXkR0lUCbeIcKc= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-redis/redis/v8 v8.11.3/go.mod h1:xNJ9xDG09FsIPwh3bWdk+0oDWHbtF9rPN0F/oD9XeKc= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= @@ -706,7 +707,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= @@ -854,15 +854,16 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -982,6 +983,8 @@ github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8t github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= github.com/rbretecher/go-postman-collection v0.9.0 h1:vXw6KBhASpz0L0igH3OsJCx5pjKbWXn9RiYMMnOO4QQ= github.com/rbretecher/go-postman-collection v0.9.0/go.mod h1:pptkyjdB/sqPycH+CCa1zrA6Wpj2Kc8Nz846qRstVVs= +github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= +github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -1313,7 +1316,6 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= diff --git a/toolkit/cache/.prettierrc.yml b/toolkit/cache/.prettierrc.yml new file mode 100644 index 00000000..8b7f044a --- /dev/null +++ b/toolkit/cache/.prettierrc.yml @@ -0,0 +1,4 @@ +semi: false +singleQuote: true +proseWrap: always +printWidth: 100 diff --git a/toolkit/cache/CHANGELOG.md b/toolkit/cache/CHANGELOG.md new file mode 100644 index 00000000..e9d6eece --- /dev/null +++ b/toolkit/cache/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +> :heart: [**Uptrace.dev** - distributed traces, logs, and errors in one place](https://uptrace.dev) + +## v8 + +- Added s2 (snappy) compression. That means that v8 can't read the data set by v7. +- Replaced LRU with TinyLFU for local cache. +- Requires go-redis v8. diff --git a/toolkit/cache/LICENSE b/toolkit/cache/LICENSE new file mode 100644 index 00000000..9a21b9aa --- /dev/null +++ b/toolkit/cache/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2016 The github.com/go-redis/cache Contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/toolkit/cache/Makefile b/toolkit/cache/Makefile new file mode 100644 index 00000000..57914e33 --- /dev/null +++ b/toolkit/cache/Makefile @@ -0,0 +1,6 @@ +all: + go test ./... + go test ./... -short -race + go test ./... -run=NONE -bench=. -benchmem + env GOOS=linux GOARCH=386 go test ./... + golangci-lint run diff --git a/toolkit/cache/README.md b/toolkit/cache/README.md new file mode 100644 index 00000000..5d31c87a --- /dev/null +++ b/toolkit/cache/README.md @@ -0,0 +1,122 @@ +# Redis cache library for Golang + +[![Build Status](https://travis-ci.org/go-redis/cache.svg)](https://travis-ci.org/go-redis/cache) +[![GoDoc](https://godoc.org/github.com/go-redis/cache?status.svg)](https://pkg.go.dev/github.com/go-redis/cache/v9?tab=doc) + +> go-redis/cache is brought to you by :star: +> [**uptrace/uptrace**](https://github.com/uptrace/uptrace). Uptrace is an open source and blazingly +> fast [distributed tracing tool](https://get.uptrace.dev/) powered by OpenTelemetry and ClickHouse. +> Give it a star as well! + +go-redis/cache library implements a cache using Redis as a key/value storage. It uses +[MessagePack](https://github.com/vmihailenco/msgpack) to marshal values. + +Optionally, you can use [TinyLFU](https://github.com/dgryski/go-tinylfu) or any other +[cache algorithm](https://github.com/vmihailenco/go-cache-benchmark) as a local in-process cache. + +If you are interested in monitoring cache hit rate, see the guide for +[Monitoring using OpenTelemetry Metrics](https://blog.uptrace.dev/posts/opentelemetry-metrics-cache-stats/). + +## Installation + +go-redis/cache supports 2 last Go versions and requires a Go version with +[modules](https://github.com/golang/go/wiki/Modules) support. So make sure to initialize a Go +module: + +```shell +go mod init github.com/my/repo +``` + +And then install go-redis/cache/v9 (note _v9_ in the import; omitting it is a popular mistake): + +```shell +go get github.com/go-redis/cache/v9 +``` + +## Quickstart + +```go +package cache_test + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" +) + +type Object struct { + Str string + Num int +} + +func Example_basicUsage() { + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "server1": ":6379", + "server2": ":6380", + }, + }) + + mycache := cache.New(&cache.Options{ + Redis: ring, + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) + + ctx := context.TODO() + key := "mykey" + obj := &Object{ + Str: "mystring", + Num: 42, + } + + if err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: obj, + TTL: time.Hour, + }); err != nil { + panic(err) + } + + var wanted Object + if err := mycache.Get(ctx, key, &wanted); err == nil { + fmt.Println(wanted) + } + + // Output: {mystring 42} +} + +func Example_advancedUsage() { + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "server1": ":6379", + "server2": ":6380", + }, + }) + + mycache := cache.New(&cache.Options{ + Redis: ring, + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) + + obj := new(Object) + err := mycache.Once(&cache.Item{ + Key: "mykey", + Value: obj, // destination + Do: func(*cache.Item) (interface{}, error) { + return &Object{ + Str: "mystring", + Num: 42, + }, nil + }, + }) + if err != nil { + panic(err) + } + fmt.Println(obj) + // Output: &{mystring 42} +} +``` diff --git a/toolkit/cache/bench_test.go b/toolkit/cache/bench_test.go new file mode 100644 index 00000000..537e3562 --- /dev/null +++ b/toolkit/cache/bench_test.go @@ -0,0 +1,58 @@ +package cache_test + +import ( + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" + "strings" + "testing" + +) + +func BenchmarkOnce(b *testing.B) { + mycache := newCacheWithLocal(newRing()) + obj := &Object{ + Str: strings.Repeat("my very large string", 10), + Num: 42, + } + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + var dst Object + err := mycache.Once(&cache.Item{ + Key: "bench-once", + Value: &dst, + Do: func(*cache.Item) (interface{}, error) { + return obj, nil + }, + }) + if err != nil { + b.Fatal(err) + } + if dst.Num != 42 { + b.Fatalf("%d != 42", dst.Num) + } + } + }) +} + +func BenchmarkSet(b *testing.B) { + mycache := newCacheWithLocal(newRing()) + obj := &Object{ + Str: strings.Repeat("my very large string", 10), + Num: 42, + } + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := mycache.Set(&cache.Item{ + Key: "bench-set", + Value: obj, + }); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/toolkit/cache/cache.go b/toolkit/cache/cache.go new file mode 100644 index 00000000..0dbc1739 --- /dev/null +++ b/toolkit/cache/cache.go @@ -0,0 +1,404 @@ +package cache + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + "github.com/klauspost/compress/s2" + "github.com/redis/go-redis/v9" + "github.com/vmihailenco/msgpack/v5" + "golang.org/x/sync/singleflight" +) + +const ( + compressionThreshold = 64 + timeLen = 4 +) + +const ( + noCompression = 0x0 + s2Compression = 0x1 +) + +var ( + ErrCacheMiss = errors.New("cache: key is missing") + errRedisLocalCacheNil = errors.New("cache: both Redis and LocalCache are nil") +) + +type rediser interface { + Set(ctx context.Context, key string, value interface{}, ttl time.Duration) *redis.StatusCmd + SetXX(ctx context.Context, key string, value interface{}, ttl time.Duration) *redis.BoolCmd + SetNX(ctx context.Context, key string, value interface{}, ttl time.Duration) *redis.BoolCmd + + Get(ctx context.Context, key string) *redis.StringCmd + Del(ctx context.Context, keys ...string) *redis.IntCmd +} + +type Item struct { + Ctx context.Context + + Key string + Value interface{} + + // TTL is the cache expiration time. + // Default TTL is 1 hour. + TTL time.Duration + + // Do returns value to be cached. + Do func(*Item) (interface{}, error) + + // SetXX only sets the key if it already exists. + SetXX bool + + // SetNX only sets the key if it does not already exist. + SetNX bool + + // SkipLocalCache skips local cache as if it is not set. + SkipLocalCache bool +} + +func (item *Item) Context() context.Context { + if item.Ctx == nil { + return context.Background() + } + return item.Ctx +} + +func (item *Item) value() (interface{}, error) { + if item.Do != nil { + return item.Do(item) + } + if item.Value != nil { + return item.Value, nil + } + return nil, nil +} + +func (item *Item) ttl() time.Duration { + return item.TTL +} + +//------------------------------------------------------------------------------ +type ( + MarshalFunc func(interface{}) ([]byte, error) + UnmarshalFunc func([]byte, interface{}) error +) + +type Options struct { + Redis rediser + LocalCache LocalCache + StatsEnabled bool + Marshal MarshalFunc + Unmarshal UnmarshalFunc +} + +type Cache struct { + opt *Options + + group singleflight.Group + + marshal MarshalFunc + unmarshal UnmarshalFunc + + hits uint64 + misses uint64 +} + +func New(opt *Options) *Cache { + cacher := &Cache{ + opt: opt, + } + + if opt.Marshal == nil { + cacher.marshal = cacher._marshal + } else { + cacher.marshal = opt.Marshal + } + + if opt.Unmarshal == nil { + cacher.unmarshal = cacher._unmarshal + } else { + cacher.unmarshal = opt.Unmarshal + } + return cacher +} + +// Set caches the item. +func (cd *Cache) Set(item *Item) error { + _, _, err := cd.set(item) + return err +} + +func (cd *Cache) set(item *Item) ([]byte, bool, error) { + value, err := item.value() + if err != nil { + return nil, false, err + } + + b, err := cd.Marshal(value) + if err != nil { + return nil, false, err + } + + if cd.opt.LocalCache != nil && !item.SkipLocalCache { + cd.opt.LocalCache.Set(item.Key, b) + } + + if cd.opt.Redis == nil { + if cd.opt.LocalCache == nil { + return b, true, errRedisLocalCacheNil + } + return b, true, nil + } + + ttl := item.ttl() + + if item.SetXX { + return b, true, cd.opt.Redis.SetXX(item.Context(), item.Key, b, ttl).Err() + } + if item.SetNX { + return b, true, cd.opt.Redis.SetNX(item.Context(), item.Key, b, ttl).Err() + } + return b, true, cd.opt.Redis.Set(item.Context(), item.Key, b, ttl).Err() +} + +// Exists reports whether value for the given key exists. +func (cd *Cache) Exists(ctx context.Context, key string) bool { + _, err := cd.getBytes(ctx, key, false) + return err == nil +} + +// Get gets the value for the given key. +func (cd *Cache) Get(ctx context.Context, key string, value interface{}) error { + return cd.get(ctx, key, value, false) +} + +// Get gets the value for the given key skipping local cache. +func (cd *Cache) GetSkippingLocalCache( + ctx context.Context, key string, value interface{}, +) error { + return cd.get(ctx, key, value, true) +} + +func (cd *Cache) get( + ctx context.Context, + key string, + value interface{}, + skipLocalCache bool, +) error { + b, err := cd.getBytes(ctx, key, skipLocalCache) + if err != nil { + return err + } + return cd.unmarshal(b, value) +} + +func (cd *Cache) getBytes(ctx context.Context, key string, skipLocalCache bool) ([]byte, error) { + if !skipLocalCache && cd.opt.LocalCache != nil { + b, ok := cd.opt.LocalCache.Get(key) + if ok { + return b, nil + } + } + + if cd.opt.Redis == nil { + if cd.opt.LocalCache == nil { + return nil, errRedisLocalCacheNil + } + return nil, ErrCacheMiss + } + + b, err := cd.opt.Redis.Get(ctx, key).Bytes() + if err != nil { + if cd.opt.StatsEnabled { + atomic.AddUint64(&cd.misses, 1) + } + if err == redis.Nil { + return nil, ErrCacheMiss + } + return nil, err + } + + if cd.opt.StatsEnabled { + atomic.AddUint64(&cd.hits, 1) + } + + if !skipLocalCache && cd.opt.LocalCache != nil { + cd.opt.LocalCache.Set(key, b) + } + return b, nil +} + +// Once gets the item.Value for the given item.Key from the cache or +// executes, caches, and returns the results of the given item.Func, +// making sure that only one execution is in-flight for a given item.Key +// at a time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +func (cd *Cache) Once(item *Item) error { + b, cached, err := cd.getSetItemBytesOnce(item) + if err != nil { + return err + } + + if item.Value == nil || len(b) == 0 { + return nil + } + + if err := cd.unmarshal(b, item.Value); err != nil { + if cached { + _ = cd.Delete(item.Context(), item.Key) + return cd.Once(item) + } + return err + } + + return nil +} + +func (cd *Cache) getSetItemBytesOnce(item *Item) (b []byte, cached bool, err error) { + if cd.opt.LocalCache != nil { + b, ok := cd.opt.LocalCache.Get(item.Key) + if ok { + return b, true, nil + } + } + + v, err, _ := cd.group.Do(item.Key, func() (interface{}, error) { + b, err := cd.getBytes(item.Context(), item.Key, item.SkipLocalCache) + if err == nil { + cached = true + return b, nil + } + + b, ok, err := cd.set(item) + if ok { + return b, nil + } + return nil, err + }) + if err != nil { + return nil, false, err + } + return v.([]byte), cached, nil +} + +func (cd *Cache) Delete(ctx context.Context, key string) error { + if cd.opt.LocalCache != nil { + cd.opt.LocalCache.Del(key) + } + + if cd.opt.Redis == nil { + if cd.opt.LocalCache == nil { + return errRedisLocalCacheNil + } + return nil + } + + _, err := cd.opt.Redis.Del(ctx, key).Result() + return err +} + +func (cd *Cache) DeleteFromLocalCache(key string) { + if cd.opt.LocalCache != nil { + cd.opt.LocalCache.Del(key) + } +} + +func (cd *Cache) Marshal(value interface{}) ([]byte, error) { + return cd.marshal(value) +} + +func (cd *Cache) _marshal(value interface{}) ([]byte, error) { + switch value := value.(type) { + case nil: + return nil, nil + case []byte: + return value, nil + case string: + return []byte(value), nil + } + + b, err := msgpack.Marshal(value) + if err != nil { + return nil, err + } + + return compress(b), nil +} + +func compress(data []byte) []byte { + if len(data) < compressionThreshold { + n := len(data) + 1 + b := make([]byte, n, n+timeLen) + copy(b, data) + b[len(b)-1] = noCompression + return b + } + + n := s2.MaxEncodedLen(len(data)) + 1 + b := make([]byte, n, n+timeLen) + b = s2.Encode(b, data) + b = append(b, s2Compression) + return b +} + +func (cd *Cache) Unmarshal(b []byte, value interface{}) error { + return cd.unmarshal(b, value) +} + +func (cd *Cache) _unmarshal(b []byte, value interface{}) error { + if len(b) == 0 { + return nil + } + + switch value := value.(type) { + case nil: + return nil + case *[]byte: + clone := make([]byte, len(b)) + copy(clone, b) + *value = clone + return nil + case *string: + *value = string(b) + return nil + } + + switch c := b[len(b)-1]; c { + case noCompression: + b = b[:len(b)-1] + case s2Compression: + b = b[:len(b)-1] + + var err error + b, err = s2.Decode(nil, b) + if err != nil { + return err + } + default: + return fmt.Errorf("unknown compression method: %x", c) + } + + return msgpack.Unmarshal(b, value) +} + +//------------------------------------------------------------------------------ + +type Stats struct { + Hits uint64 + Misses uint64 +} + +// Stats returns cache statistics. +func (cd *Cache) Stats() *Stats { + if !cd.opt.StatsEnabled { + return nil + } + return &Stats{ + Hits: atomic.LoadUint64(&cd.hits), + Misses: atomic.LoadUint64(&cd.misses), + } +} diff --git a/toolkit/cache/cache_test.go b/toolkit/cache/cache_test.go new file mode 100644 index 00000000..a7a2ab1e --- /dev/null +++ b/toolkit/cache/cache_test.go @@ -0,0 +1,426 @@ +package cache_test + +import ( + "context" + "errors" + "io" + "sync" + "sync/atomic" + "testing" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/redis/go-redis/v9" + + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" +) + +func TestGinkgo(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "cache") +} + +func perform(n int, cbs ...func(int)) { + var wg sync.WaitGroup + for _, cb := range cbs { + for i := 0; i < n; i++ { + wg.Add(1) + go func(cb func(int), i int) { + defer wg.Done() + defer GinkgoRecover() + + cb(i) + }(cb, i) + } + } + wg.Wait() +} + +var _ = Describe("Cache", func() { + ctx := context.TODO() + + const key = "mykey" + var obj *Object + + var rdb *redis.Ring + var mycache *cache.Cache + + testCache := func() { + It("Gets and Sets nil", func() { + err := mycache.Set(&cache.Item{ + Key: key, + TTL: time.Hour, + }) + Expect(err).NotTo(HaveOccurred()) + + err = mycache.Get(ctx, key, nil) + Expect(err).NotTo(HaveOccurred()) + + Expect(mycache.Exists(ctx, key)).To(BeTrue()) + }) + + It("Deletes key", func() { + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + TTL: time.Hour, + }) + Expect(err).NotTo(HaveOccurred()) + + Expect(mycache.Exists(ctx, key)).To(BeTrue()) + + err = mycache.Delete(ctx, key) + Expect(err).NotTo(HaveOccurred()) + + err = mycache.Get(ctx, key, nil) + Expect(err).To(Equal(cache.ErrCacheMiss)) + + Expect(mycache.Exists(ctx, key)).To(BeFalse()) + }) + + It("Gets and Sets data", func() { + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: obj, + TTL: time.Hour, + }) + Expect(err).NotTo(HaveOccurred()) + + wanted := new(Object) + err = mycache.Get(ctx, key, wanted) + Expect(err).NotTo(HaveOccurred()) + Expect(wanted).To(Equal(obj)) + + Expect(mycache.Exists(ctx, key)).To(BeTrue()) + }) + + It("Sets string as is", func() { + value := "str_value" + + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: value, + }) + Expect(err).NotTo(HaveOccurred()) + + var dst string + err = mycache.Get(ctx, key, &dst) + Expect(err).NotTo(HaveOccurred()) + Expect(dst).To(Equal(value)) + }) + + It("Sets bytes as is", func() { + value := []byte("str_value") + + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: value, + }) + Expect(err).NotTo(HaveOccurred()) + + var dst []byte + err = mycache.Get(ctx, key, &dst) + Expect(err).NotTo(HaveOccurred()) + Expect(dst).To(Equal(value)) + }) + + It("can be used with Incr", func() { + if rdb == nil { + return + } + + value := "123" + + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: value, + }) + Expect(err).NotTo(HaveOccurred()) + + n, err := rdb.Incr(ctx, key).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(int64(124))) + }) + + Describe("Once func", func() { + It("calls Func when cache fails", func() { + err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: int64(0), + }) + Expect(err).NotTo(HaveOccurred()) + + var got bool + err = mycache.Get(ctx, key, &got) + Expect(err).To(MatchError("msgpack: invalid code=d3 decoding bool")) + + err = mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &got, + Do: func(*cache.Item) (interface{}, error) { + return true, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeTrue()) + + got = false + err = mycache.Get(ctx, key, &got) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeTrue()) + }) + + It("does not cache when Func fails", func() { + perform(100, func(int) { + var got bool + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &got, + Do: func(*cache.Item) (interface{}, error) { + return nil, io.EOF + }, + }) + Expect(err).To(Equal(io.EOF)) + Expect(got).To(BeFalse()) + }) + + var got bool + err := mycache.Get(ctx, key, &got) + Expect(err).To(Equal(cache.ErrCacheMiss)) + + err = mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &got, + Do: func(*cache.Item) (interface{}, error) { + return true, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeTrue()) + }) + + It("works with Value", func() { + var callCount int64 + perform(100, func(int) { + got := new(Object) + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: got, + Do: func(*cache.Item) (interface{}, error) { + atomic.AddInt64(&callCount, 1) + return obj, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(obj)) + }) + Expect(callCount).To(Equal(int64(1))) + }) + + It("works with ptr and non-ptr", func() { + var callCount int64 + perform(100, func(int) { + got := new(Object) + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: got, + Do: func(*cache.Item) (interface{}, error) { + atomic.AddInt64(&callCount, 1) + return *obj, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(obj)) + }) + Expect(callCount).To(Equal(int64(1))) + }) + + It("works with bool", func() { + var callCount int64 + perform(100, func(int) { + var got bool + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &got, + Do: func(*cache.Item) (interface{}, error) { + atomic.AddInt64(&callCount, 1) + return true, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeTrue()) + }) + Expect(callCount).To(Equal(int64(1))) + }) + + It("works without Value and nil result", func() { + var callCount int64 + perform(100, func(int) { + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Do: func(*cache.Item) (interface{}, error) { + atomic.AddInt64(&callCount, 1) + return nil, nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + }) + Expect(callCount).To(Equal(int64(1))) + }) + + It("works without Value and error result", func() { + var callCount int64 + perform(100, func(int) { + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Do: func(*cache.Item) (interface{}, error) { + time.Sleep(100 * time.Millisecond) + atomic.AddInt64(&callCount, 1) + return nil, errors.New("error stub") + }, + }) + Expect(err).To(MatchError("error stub")) + }) + Expect(callCount).To(Equal(int64(1))) + }) + + It("does not cache error result", func() { + var callCount int64 + do := func(sleep time.Duration) (int, error) { + var n int + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &n, + Do: func(*cache.Item) (interface{}, error) { + time.Sleep(sleep) + + n := atomic.AddInt64(&callCount, 1) + if n == 1 { + return nil, errors.New("error stub") + } + return 42, nil + }, + }) + if err != nil { + return 0, err + } + return n, nil + } + + perform(100, func(int) { + n, err := do(100 * time.Millisecond) + Expect(err).To(MatchError("error stub")) + Expect(n).To(Equal(0)) + }) + + perform(100, func(int) { + n, err := do(0) + Expect(err).NotTo(HaveOccurred()) + Expect(n).To(Equal(42)) + }) + + Expect(callCount).To(Equal(int64(2))) + }) + + It("skips Set when TTL = -1", func() { + key := "skip-set" + + var value string + err := mycache.Once(&cache.Item{ + Ctx: ctx, + Key: key, + Value: &value, + Do: func(item *cache.Item) (interface{}, error) { + item.TTL = -1 + return "hello", nil + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(value).To(Equal("hello")) + + if rdb != nil { + exists, err := rdb.Exists(ctx, key).Result() + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(Equal(int64(0))) + } + }) + }) + } + + BeforeEach(func() { + obj = &Object{ + Str: "mystring", + Num: 42, + } + }) + + Context("without LocalCache", func() { + BeforeEach(func() { + rdb = newRing() + mycache = newCache(rdb) + }) + + testCache() + }) + + Context("with LocalCache", func() { + BeforeEach(func() { + rdb = newRing() + mycache = newCacheWithLocal(rdb) + }) + + testCache() + }) + + Context("with LocalCache and without Redis", func() { + BeforeEach(func() { + rdb = nil + mycache = cache.New(&cache.Options{ + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) + }) + + testCache() + }) +}) + +func newRing() *redis.Ring { + ctx := context.TODO() + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "server1": ":6379", + }, + }) + _ = ring.ForEachShard(ctx, func(ctx context.Context, client *redis.Client) error { + return client.FlushDB(ctx).Err() + }) + return ring +} + +func newCache(rdb *redis.Ring) *cache.Cache { + return cache.New(&cache.Options{ + Redis: rdb, + }) +} + +func newCacheWithLocal(rdb *redis.Ring) *cache.Cache { + return cache.New(&cache.Options{ + Redis: rdb, + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) +} diff --git a/toolkit/cache/example_cache_test.go b/toolkit/cache/example_cache_test.go new file mode 100644 index 00000000..b42abc0e --- /dev/null +++ b/toolkit/cache/example_cache_test.go @@ -0,0 +1,84 @@ +package cache_test + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" +) + +type Object struct { + Str string + Num int +} + +func Example_basicUsage() { + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "server1": ":6379", + "server2": ":6380", + }, + }) + + mycache := cache.New(&cache.Options{ + Redis: ring, + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) + + ctx := context.TODO() + key := "mykey" + obj := &Object{ + Str: "mystring", + Num: 42, + } + + if err := mycache.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: obj, + TTL: time.Hour, + }); err != nil { + panic(err) + } + + var wanted Object + if err := mycache.Get(ctx, key, &wanted); err == nil { + fmt.Println(wanted) + } + + // Output: {mystring 42} +} + +func Example_advancedUsage() { + ring := redis.NewRing(&redis.RingOptions{ + Addrs: map[string]string{ + "server1": ":6379", + "server2": ":6380", + }, + }) + + mycache := cache.New(&cache.Options{ + Redis: ring, + LocalCache: cache.NewTinyLFU(1000, time.Minute), + }) + + obj := new(Object) + err := mycache.Once(&cache.Item{ + Key: "mykey", + Value: obj, // destination + Do: func(*cache.Item) (interface{}, error) { + return &Object{ + Str: "mystring", + Num: 42, + }, nil + }, + }) + if err != nil { + panic(err) + } + fmt.Println(obj) + // Output: &{mystring 42} +} diff --git a/toolkit/cache/local.go b/toolkit/cache/local.go new file mode 100644 index 00000000..81f50978 --- /dev/null +++ b/toolkit/cache/local.go @@ -0,0 +1,81 @@ +package cache + +import ( + "math/rand" + "sync" + "time" + + "github.com/vmihailenco/go-tinylfu" +) + +type LocalCache interface { + Set(key string, data []byte) + Get(key string) ([]byte, bool) + Del(key string) +} + +type TinyLFU struct { + mu sync.Mutex + rand *rand.Rand + lfu *tinylfu.T + ttl time.Duration + offset time.Duration +} + +var _ LocalCache = (*TinyLFU)(nil) + +func NewTinyLFU(size int, ttl time.Duration) *TinyLFU { + const maxOffset = 10 * time.Second + + offset := ttl / 10 + if offset > maxOffset { + offset = maxOffset + } + + return &TinyLFU{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + lfu: tinylfu.New(size, 100000), + ttl: ttl, + offset: offset, + } +} + +func (c *TinyLFU) UseRandomizedTTL(offset time.Duration) { + c.offset = offset +} + +func (c *TinyLFU) Set(key string, b []byte) { + c.mu.Lock() + defer c.mu.Unlock() + + ttl := c.ttl + if c.offset > 0 { + ttl += time.Duration(c.rand.Int63n(int64(c.offset))) + } + + c.lfu.Set(&tinylfu.Item{ + Key: key, + Value: b, + ExpireAt: time.Now().Add(ttl), + }) +} + +func (c *TinyLFU) Get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + + val, ok := c.lfu.Get(key) + if !ok { + return nil, false + } + + b := val.([]byte) + return b, true +} + +func (c *TinyLFU) Del(key string) { + c.mu.Lock() + defer c.mu.Unlock() + + c.lfu.Del(key) +} diff --git a/toolkit/cache/local_test.go b/toolkit/cache/local_test.go new file mode 100644 index 00000000..f8dad079 --- /dev/null +++ b/toolkit/cache/local_test.go @@ -0,0 +1,56 @@ +package cache_test + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" +) + +func TestTinyLFU_Get_CorruptionOnExpiry(t *testing.T) { + strFor := func(i int) string { + return fmt.Sprintf("a string %d", i) + } + keyName := func(i int) string { + return fmt.Sprintf("key-%00000d", i) + } + + mycache := cache.NewTinyLFU(1000, 1*time.Second) + size := 50000 + // Put a bunch of stuff in the cache with a TTL of 1 second + for i := 0; i < size; i++ { + key := keyName(i) + mycache.Set(key, []byte(strFor(i))) + } + + // Read stuff for a bit longer than the TTL - that's when the corruption occurs + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + done := ctx.Done() +loop: + for { + select { + case <-done: + // this is expected + break loop + default: + i := rand.Intn(size) + key := keyName(i) + + b, ok := mycache.Get(key) + if !ok { + continue loop + } + + got := string(b) + expected := strFor(i) + if got != expected { + t.Fatalf("expected=%q got=%q key=%q", expected, got, key) + } + } + } +} diff --git a/toolkit/gormgen/tests/go.mod b/toolkit/gormgen/tests/go.mod index ddacf95e..efe4e74f 100644 --- a/toolkit/gormgen/tests/go.mod +++ b/toolkit/gormgen/tests/go.mod @@ -4,14 +4,13 @@ go 1.16 require ( github.com/mattn/go-sqlite3 v1.14.16 // indirect - golang.org/x/tools v0.5.0 // indirect - gorm.io/driver/mysql v1.4.5 + github.com/unionj-cloud/go-doudou/v2 v2.1.9-0.20230825031202-41ef70f1be6f + gorm.io/driver/mysql v1.5.1-0.20230509030346-3715c134c25b gorm.io/driver/sqlite v1.4.4 gorm.io/gen v0.3.19 - gorm.io/gorm v1.24.3 + gorm.io/gorm v1.25.1-0.20230505075827-e61b98d69677 gorm.io/hints v1.1.1 // indirect gorm.io/plugin/dbresolver v1.4.0 - github.com/unionj-cloud/go-doudou/v2 main ) -replace github.com/unionj-cloud/go-doudou/v2 main => ../../../ +replace github.com/unionj-cloud/go-doudou/v2 v2.1.9-0.20230825031202-41ef70f1be6f => ../../../ diff --git a/toolkit/sqlext/logger/logger.go b/toolkit/sqlext/logger/logger.go index 57817a60..c002333b 100644 --- a/toolkit/sqlext/logger/logger.go +++ b/toolkit/sqlext/logger/logger.go @@ -14,7 +14,6 @@ import ( "github.com/unionj-cloud/go-doudou/v2/toolkit/stringutils" "github.com/unionj-cloud/go-doudou/v2/toolkit/zlogger" "os" - "regexp" "strings" ) @@ -51,32 +50,18 @@ func NewSqlLogger(opts ...SqlLoggerOption) SqlLogger { return sqlLogger } -var limitre *regexp.Regexp - -func init() { - limitre = regexp.MustCompile(`limit '\d+'(,'\d+')?`) -} - func PopulatedSql(query string, args ...interface{}) string { - query = strings.Join(strings.Fields(query), " ") - copiedArgs := make([]interface{}, len(args)) - copy(copiedArgs, args) - for i, arg := range copiedArgs { - if arg == nil { - continue - } + var sb strings.Builder + sb.WriteString(strings.Join(strings.Fields(query), " ")) + for _, arg := range args { value := reflectutils.ValueOf(arg) if value.IsValid() { - copiedArgs[i] = value.Interface() + sb.WriteString(fmt.Sprint(value.Interface())) + } else { + sb.WriteString(fmt.Sprint(arg)) } } - str := strings.ReplaceAll(fmt.Sprintf(strings.ReplaceAll(query, "?", "'%v'"), copiedArgs...), "''", "null") - if limitre.MatchString(str) { - str = limitre.ReplaceAllStringFunc(str, func(s string) string { - return strings.ReplaceAll(s, "'", "") - }) - } - return str + return strings.ReplaceAll(sb.String(), "''", "null") } func (receiver SqlLogger) LogWithErr(ctx context.Context, err error, hit *bool, query string, args ...interface{}) { diff --git a/toolkit/sqlext/wrapper/wrapper.go b/toolkit/sqlext/wrapper/wrapper.go index db133db5..94c764d5 100644 --- a/toolkit/sqlext/wrapper/wrapper.go +++ b/toolkit/sqlext/wrapper/wrapper.go @@ -3,10 +3,10 @@ package wrapper import ( "context" "database/sql" - "github.com/go-redis/cache/v8" "github.com/jmoiron/sqlx" "github.com/lithammer/shortuuid/v4" "github.com/pkg/errors" + "github.com/unionj-cloud/go-doudou/v2/toolkit/cache" "github.com/unionj-cloud/go-doudou/v2/toolkit/caller" "github.com/unionj-cloud/go-doudou/v2/toolkit/sqlext/logger" "time" @@ -34,6 +34,7 @@ type Querier interface { Rebind(query string) string BindNamed(query string, arg interface{}) (string, []interface{}, error) SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + //RefreshCache() } // GddDB wraps sqlx.DB