diff --git a/toolkit/caches/LICENSE b/toolkit/caches/LICENSE new file mode 100644 index 00000000..ed43104c --- /dev/null +++ b/toolkit/caches/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023-NOW Kristian Tsivkov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkit/caches/README.md b/toolkit/caches/README.md new file mode 100644 index 00000000..b7aaba22 --- /dev/null +++ b/toolkit/caches/README.md @@ -0,0 +1,235 @@ +# Gorm Caches + +Gorm Caches plugin using database request reductions (easer), and response caching mechanism provide you an easy way to optimize database performance. + +## Features + +- Database request reduction. If three identical requests are running at the same time, only the first one is going to be executed, and its response will be returned for all. +- Database response caching. By implementing the Cacher interface, you can easily setup a caching mechanism for your database queries. +- Supports all databases that are supported by gorm itself. + +## Install + +```bash +go get -u github.com/go-gorm/caches/v2 +``` + +## Usage + +Configure the `easer`, and the `cacher`, and then load the plugin to gorm. + +```go +package main + +import ( + "fmt" + "sync" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Easer: true, + Cacher: &yourCacherImplementation{}, + }} + _ = db.Use(cachesPlugin) +} +``` + +## Easer Example + +```go +package main + +import ( + "fmt" + "sync" + "time" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type UserRoleModel struct { + gorm.Model + Name string `gorm:"unique"` +} + +type UserModel struct { + gorm.Model + Name string + RoleId uint + Role *UserRoleModel `gorm:"foreignKey:role_id;references:id"` +} + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Easer: true, + }} + + _ = db.Use(cachesPlugin) + + _ = db.AutoMigrate(&UserRoleModel{}) + + _ = db.AutoMigrate(&UserModel{}) + + adminRole := &UserRoleModel{ + Name: "Admin", + } + db.FirstOrCreate(adminRole, "Name = ?", "Admin") + + guestRole := &UserRoleModel{ + Name: "Guest", + } + db.FirstOrCreate(guestRole, "Name = ?", "Guest") + + db.Save(&UserModel{ + Name: "ktsivkov", + Role: adminRole, + }) + db.Save(&UserModel{ + Name: "anonymous", + Role: guestRole, + }) + + var ( + q1Users []UserModel + q2Users []UserModel + ) + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + db.Model(&UserModel{}).Joins("Role").Find(&q1Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + db.Model(&UserModel{}).Joins("Role").Find(&q2Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + wg.Done() + }() + wg.Wait() + + fmt.Println(fmt.Sprintf("%+v", q1Users)) + fmt.Println(fmt.Sprintf("%+v", q2Users)) +} +``` + +## Cacher Example + +```go +package main + +import ( + "fmt" + "sync" + + "github.com/go-gorm/caches" + "gorm.io/driver/mysql" + "gorm.io/gorm" +) + +type UserRoleModel struct { + gorm.Model + Name string `gorm:"unique"` +} + +type UserModel struct { + gorm.Model + Name string + RoleId uint + Role *UserRoleModel `gorm:"foreignKey:role_id;references:id"` +} + +type dummyCacher struct { + store *sync.Map +} + +func (c *dummyCacher) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *dummyCacher) Get(key string) *caches.Query { + c.init() + val, ok := c.store.Load(key) + if !ok { + return nil + } + + return val.(*caches.Query) +} + +func (c *dummyCacher) Store(key string, val *caches.Query) error { + c.init() + c.store.Store(key, val) + return nil +} + +func main() { + db, _ := gorm.Open( + mysql.Open("DATABASE_DSN"), + &gorm.Config{}, + ) + + cachesPlugin := &caches.Caches{Conf: &caches.Config{ + Cacher: &dummyCacher{}, + }} + + _ = db.Use(cachesPlugin) + + _ = db.AutoMigrate(&UserRoleModel{}) + + _ = db.AutoMigrate(&UserModel{}) + + adminRole := &UserRoleModel{ + Name: "Admin", + } + db.FirstOrCreate(adminRole, "Name = ?", "Admin") + + guestRole := &UserRoleModel{ + Name: "Guest", + } + db.FirstOrCreate(guestRole, "Name = ?", "Guest") + + db.Save(&UserModel{ + Name: "ktsivkov", + Role: adminRole, + }) + db.Save(&UserModel{ + Name: "anonymous", + Role: guestRole, + }) + + var ( + q1Users []UserModel + q2Users []UserModel + ) + + db.Model(&UserModel{}).Joins("Role").Find(&q1Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + fmt.Println(fmt.Sprintf("%+v", q1Users)) + + db.Model(&UserModel{}).Joins("Role").Find(&q2Users, "Role.Name = ? AND Sleep(1) = false", "Admin") + fmt.Println(fmt.Sprintf("%+v", q2Users)) +} +``` + +## License + +MIT license. + +## Easer +The easer is an adjusted version of the [ServantGo](https://github.com/ktsivkov/servantgo) library to fit the needs of this plugin. diff --git a/toolkit/caches/cacher.go b/toolkit/caches/cacher.go new file mode 100644 index 00000000..44536aba --- /dev/null +++ b/toolkit/caches/cacher.go @@ -0,0 +1,7 @@ +package caches + +type Cacher interface { + Get(key string) *Query + Store(key string, val *Query) error + Delete(tag string, tags ...string) error +} diff --git a/toolkit/caches/cacher_test.go b/toolkit/caches/cacher_test.go new file mode 100644 index 00000000..b0d836d0 --- /dev/null +++ b/toolkit/caches/cacher_test.go @@ -0,0 +1,66 @@ +package caches + +import ( + "errors" + "sync" +) + +type cacherMock struct { + store *sync.Map +} + +func (c *cacherMock) Delete(tag string, tags ...string) error { + //TODO implement me + panic("implement me") +} + +func (c *cacherMock) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *cacherMock) Get(key string) *Query { + c.init() + val, ok := c.store.Load(key) + if !ok { + return nil + } + + return val.(*Query) +} + +func (c *cacherMock) Store(key string, val *Query) error { + c.init() + c.store.Store(key, val) + return nil +} + +type cacherStoreErrorMock struct { + store *sync.Map +} + +func (c *cacherStoreErrorMock) Delete(tag string, tags ...string) error { + //TODO implement me + panic("implement me") +} + +func (c *cacherStoreErrorMock) init() { + if c.store == nil { + c.store = &sync.Map{} + } +} + +func (c *cacherStoreErrorMock) Get(key string) *Query { + c.init() + val, ok := c.store.Load(key) + if !ok { + return nil + } + + return val.(*Query) +} + +func (c *cacherStoreErrorMock) Store(string, *Query) error { + return errors.New("store-error") +} diff --git a/toolkit/caches/caches.go b/toolkit/caches/caches.go new file mode 100644 index 00000000..d91706e3 --- /dev/null +++ b/toolkit/caches/caches.go @@ -0,0 +1,191 @@ +package caches + +import ( + "gorm.io/gorm/callbacks" + "gorm.io/gorm/clause" + "sync" + + "gorm.io/gorm" +) + +type Caches struct { + Conf *Config + + queue *sync.Map + queryCb func(*gorm.DB) +} + +type Config struct { + Easer bool + Cacher Cacher +} + +func (c *Caches) Name() string { + return "gorm:caches" +} + +func (c *Caches) Initialize(db *gorm.DB) error { + if c.Conf == nil { + c.Conf = &Config{ + Easer: false, + Cacher: nil, + } + } + + if c.Conf.Easer { + c.queue = &sync.Map{} + } + + c.queryCb = db.Callback().Query().Get("gorm:query") + + err := db.Callback().Query().Replace("gorm:query", c.Query) + if err != nil { + return err + } + + err = db.Callback().Create().After("gorm:after_create").Register("cache:after_create", c.AfterWrite) + if err != nil { + return err + } + + err = db.Callback().Create().After("gorm:after_delete").Register("cache:after_delete", c.AfterWrite) + if err != nil { + return err + } + + err = db.Callback().Create().After("gorm:after_update").Register("cache:after_update", c.AfterWrite) + if err != nil { + return err + } + + return nil +} + +func (c *Caches) Query(db *gorm.DB) { + if c.Conf.Easer == false && c.Conf.Cacher == nil { + c.queryCb(db) + return + } + + identifier := buildIdentifier(db) + + if c.checkCache(db, identifier) { + return + } + + c.ease(db, identifier) + if db.Error != nil { + return + } + + c.storeInCache(db, identifier) + if db.Error != nil { + return + } +} + +func (c *Caches) AfterWrite(db *gorm.DB) { + if c.Conf.Easer == false && c.Conf.Cacher == nil { + return + } + + callbacks.BuildQuerySQL(db) + + tables := getTables(db) + if len(tables) == 1 { + c.deleteCache(db, tables[0]) + } else { + c.deleteCache(db, tables[0], tables[1:]...) + } + + if db.Error != nil { + return + } +} + +func (c *Caches) ease(db *gorm.DB, identifier string) { + if c.Conf.Easer == false { + c.queryCb(db) + return + } + + res := ease(&queryTask{ + id: identifier, + db: db, + queryCb: c.queryCb, + }, c.queue).(*queryTask) + + if db.Error != nil { + return + } + + if res.db.Statement.Dest == db.Statement.Dest { + return + } + + q := Query{ + Dest: db.Statement.Dest, + RowsAffected: db.Statement.RowsAffected, + } + q.replaceOn(res.db) +} + +func (c *Caches) checkCache(db *gorm.DB, identifier string) bool { + if c.Conf.Cacher != nil { + if res := c.Conf.Cacher.Get(identifier); res != nil { + res.replaceOn(db) + return true + } + } + return false +} + +func getFromClause(db *gorm.DB) *clause.From { + if db == nil || db.Statement == nil { + return &clause.From{} + } + c, ok := db.Statement.Clauses[clause.From{}.Name()] + if !ok || c.Expression == nil { + return &clause.From{} + } + from, ok := c.Expression.(clause.From) + if !ok { + return &clause.From{} + } + return &from +} + +func getTables(db *gorm.DB) []string { + // Find all table names within the sql statement as cache tags + from := getFromClause(db) + var tables []string + for _, item := range from.Tables { + tables = append(tables, item.Name) + } + for _, item := range from.Joins { + tables = append(tables, item.Table.Name) + } + return tables +} + +func (c *Caches) storeInCache(db *gorm.DB, identifier string) { + if c.Conf.Cacher != nil { + err := c.Conf.Cacher.Store(identifier, &Query{ + Tags: getTables(db), + Dest: db.Statement.Dest, + RowsAffected: db.Statement.RowsAffected, + }) + if err != nil { + _ = db.AddError(err) + } + } +} + +func (c *Caches) deleteCache(db *gorm.DB, tag string, tags ...string) { + if c.Conf.Cacher != nil { + err := c.Conf.Cacher.Delete(tag, tags...) + if err != nil { + _ = db.AddError(err) + } + } +} diff --git a/toolkit/caches/caches_test.go b/toolkit/caches/caches_test.go new file mode 100644 index 00000000..58a337f4 --- /dev/null +++ b/toolkit/caches/caches_test.go @@ -0,0 +1,423 @@ +package caches + +import ( + "fmt" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "gorm.io/gorm" + "gorm.io/gorm/utils/tests" +) + +type mockDest struct { + Result string +} + +func TestCaches_Name(t *testing.T) { + caches := &Caches{ + Conf: &Config{ + Easer: true, + Cacher: nil, + }, + } + expectedName := "gorm:caches" + if act := caches.Name(); act != expectedName { + t.Errorf("Name on caches did not return the expected value, expected: %s, actual: %s", + expectedName, act) + } +} + +func TestCaches_Initialize(t *testing.T) { + t.Run("empty config", func(t *testing.T) { + caches := &Caches{} + db, err := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + if err != nil { + t.Fatalf("gorm initialization resulted into an unexpected error, %s", err.Error()) + } + + originalQueryCb := db.Callback().Query().Get("gorm:query") + + err = db.Use(caches) + if err != nil { + t.Fatalf("gorm:caches loading resulted into an unexpected error, %s", err.Error()) + } + + newQueryCallback := db.Callback().Query().Get("gorm:query") + + if reflect.ValueOf(originalQueryCb).Pointer() == reflect.ValueOf(newQueryCallback).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback") + } + + if reflect.ValueOf(newQueryCallback).Pointer() != reflect.ValueOf(caches.Query).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback, with caches.Query") + } + + if reflect.ValueOf(originalQueryCb).Pointer() != reflect.ValueOf(caches.queryCb).Pointer() { + t.Errorf("loading of gorm:caches, expected to load original `gorm:query` callback, to caches.queryCb") + } + }) + t.Run("config - easer", func(t *testing.T) { + caches := &Caches{ + Conf: &Config{ + Easer: true, + Cacher: nil, + }, + } + db, err := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + if err != nil { + t.Fatalf("gorm initialization resulted into an unexpected error, %s", err.Error()) + } + + originalQueryCb := db.Callback().Query().Get("gorm:query") + + err = db.Use(caches) + if err != nil { + t.Fatalf("gorm:caches loading resulted into an unexpected error, %s", err.Error()) + } + + newQueryCallback := db.Callback().Query().Get("gorm:query") + + if reflect.ValueOf(originalQueryCb).Pointer() == reflect.ValueOf(newQueryCallback).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback") + } + + if reflect.ValueOf(newQueryCallback).Pointer() != reflect.ValueOf(caches.Query).Pointer() { + t.Errorf("loading of gorm:caches, expected to replace the `gorm:query` callback, with caches.Query") + } + + if reflect.ValueOf(originalQueryCb).Pointer() != reflect.ValueOf(caches.queryCb).Pointer() { + t.Errorf("loading of gorm:caches, expected to load original `gorm:query` callback, to caches.queryCb") + } + }) +} + +func TestCaches_Query(t *testing.T) { + t.Run("nothing enabled", func(t *testing.T) { + conf := &Config{ + Easer: false, + Cacher: nil, + } + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + caches := &Caches{ + Conf: conf, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + + t.Run("easer only", func(t *testing.T) { + conf := &Config{ + Easer: true, + Cacher: nil, + } + + t.Run("one query", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + + t.Run("two identical queries", func(t *testing.T) { + t.Run("without error", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery := "demo-query" + db1.Statement.SQL.WriteString(exampleQuery) + db2.Statement.SQL.WriteString(exampleQuery) + + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + caches.Query(db1) // Execute the query + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) // Execute the second query half a second later + caches.Query(db2) // Execute the query + wg.Done() + }() + wg.Wait() + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 1 { + t.Errorf("when executing two identical queries, expected to run %d time, but %d", 1, act) + } + }) + }) + + t.Run("two different queries", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: conf, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery1 := "demo-query-1" + db1.Statement.SQL.WriteString(exampleQuery1) + exampleQuery2 := "demo-query-2" + db2.Statement.SQL.WriteString(exampleQuery2) + + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + caches.Query(db1) // Execute the query + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) // Execute the second query half a second later + caches.Query(db2) // Execute the query + wg.Done() + }() + wg.Wait() + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 2 { + t.Errorf("when executing two identical queries, expected to run %d times, but %d", 2, act) + } + }) + }) + + t.Run("cacher only", func(t *testing.T) { + t.Run("one query", func(t *testing.T) { + t.Run("with error", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherStoreErrorMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error == nil { + t.Error("an error was expected, got none") + } + }) + + t.Run("without error", func(t *testing.T) { + db, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + db.Statement.Dest.(*mockDest).Result = db.Statement.SQL.String() + }, + } + + // Set the query SQL into something specific + exampleQuery := "demo-query" + db.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db) // Execute the query + + if db.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db.Error) + } + + if db.Statement.Dest == nil { + t.Fatal("no query result was set after caches Query was executed") + } + + if res := db.Statement.Dest.(*mockDest); res.Result != exampleQuery { + t.Errorf("the execution of the Query expected a result of `%s`, got `%s`", exampleQuery, res) + } + }) + }) + + t.Run("two identical queries", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery := "demo-query" + db1.Statement.SQL.WriteString(exampleQuery) + db2.Statement.SQL.WriteString(exampleQuery) + + caches.Query(db1) + caches.Query(db2) + + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 1 { + t.Errorf("when executing two identical queries, expected to run %d time, but %d", 1, act) + } + }) + + t.Run("two different queries", func(t *testing.T) { + var incr int32 + db1, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db1.Statement.Dest = &mockDest{} + db2, _ := gorm.Open(tests.DummyDialector{}, &gorm.Config{}) + db2.Statement.Dest = &mockDest{} + + caches := &Caches{ + Conf: &Config{ + Easer: false, + Cacher: &cacherMock{}, + }, + + queue: &sync.Map{}, + queryCb: func(db *gorm.DB) { + time.Sleep(1 * time.Second) + atomic.AddInt32(&incr, 1) + + db.Statement.Dest.(*mockDest).Result = fmt.Sprintf("%d", atomic.LoadInt32(&incr)) + }, + } + + // Set the queries' SQL into something specific + exampleQuery1 := "demo-query-1" + db1.Statement.SQL.WriteString(exampleQuery1) + exampleQuery2 := "demo-query-2" + db2.Statement.SQL.WriteString(exampleQuery2) + + caches.Query(db1) + if db1.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db1.Error) + } + + caches.Query(db2) + if db2.Error != nil { + t.Fatalf("an unexpected error has occurred, %v", db2.Error) + } + + if act := atomic.LoadInt32(&incr); act != 2 { + t.Errorf("when executing two identical queries, expected to run %d times, but %d", 2, act) + } + }) + }) +} diff --git a/toolkit/caches/easer.go b/toolkit/caches/easer.go new file mode 100644 index 00000000..b3135442 --- /dev/null +++ b/toolkit/caches/easer.go @@ -0,0 +1,30 @@ +package caches + +import "sync" + +func ease(t task, queue *sync.Map) task { + eq := &eased{ + task: t, + wg: &sync.WaitGroup{}, + } + eq.wg.Add(1) + + runner, ok := queue.LoadOrStore(t.GetId(), eq) + et := runner.(*eased) + + // If this request is the first of its kind, we execute the Run + if !ok { + et.task.Run() + + queue.Delete(et.task.GetId()) + et.wg.Done() + } + + et.wg.Wait() + return et.task +} + +type eased struct { + task task + wg *sync.WaitGroup +} diff --git a/toolkit/caches/easer_test.go b/toolkit/caches/easer_test.go new file mode 100644 index 00000000..c2b49e53 --- /dev/null +++ b/toolkit/caches/easer_test.go @@ -0,0 +1,102 @@ +package caches + +import ( + "sync" + "testing" + "time" +) + +func TestEase(t *testing.T) { + t.Run("same queries", func(t *testing.T) { + queue := &sync.Map{} + + myTask := &mockTask{ + delay: 1 * time.Second, + expRes: "expect-this", + id: "unique-id", + } + myDupTask := &mockTask{ + delay: 1 * time.Second, + expRes: "not-this", + id: "unique-id", + } + + wg := &sync.WaitGroup{} + wg.Add(2) + + var ( + myTaskRes *mockTask + myDupTaskRes *mockTask + ) + + // Both queries will run at the same time, the second one will run half a second later + go func() { + myTaskRes = ease(myTask, queue).(*mockTask) + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + myDupTaskRes = ease(myDupTask, queue).(*mockTask) + wg.Done() + }() + wg.Wait() + + if myTaskRes.actRes != myTaskRes.expRes { + t.Error("expected first query to be executed") + } + + if myTaskRes.actRes != myDupTaskRes.actRes { + t.Errorf("expected same result from both tasks, expected: %s, actual: %s", + myTaskRes.actRes, myDupTaskRes.actRes) + } + }) + + t.Run("different queries", func(t *testing.T) { + queue := &sync.Map{} + + myTask := &mockTask{ + delay: 1 * time.Second, + expRes: "expect-this", + id: "unique-id", + } + myDupTask := &mockTask{ + delay: 1 * time.Second, + expRes: "not-this", + id: "other-unique-id", + } + + wg := &sync.WaitGroup{} + wg.Add(2) + + var ( + myTaskRes *mockTask + myDupTaskRes *mockTask + ) + + // Both queries will run at the same time, the second one will run half a second later + go func() { + myTaskRes = ease(myTask, queue).(*mockTask) + wg.Done() + }() + go func() { + time.Sleep(500 * time.Millisecond) + myDupTaskRes = ease(myDupTask, queue).(*mockTask) + wg.Done() + }() + wg.Wait() + + if myTaskRes.actRes != myTaskRes.expRes { + t.Errorf("expected first query to be executed, expected: %s, actual: %s", + myTaskRes.actRes, myTaskRes.expRes) + } + + if myTaskRes.actRes == myDupTaskRes.actRes { + t.Errorf("expected different result from both tasks, expected: %s, actual: %s", + myTaskRes.actRes, myDupTaskRes.actRes) + } + + if myDupTaskRes.actRes != myDupTaskRes.expRes { + t.Error("expected second query to be executed") + } + }) +} diff --git a/toolkit/caches/identifier.go b/toolkit/caches/identifier.go new file mode 100644 index 00000000..a06128d8 --- /dev/null +++ b/toolkit/caches/identifier.go @@ -0,0 +1,26 @@ +package caches + +import ( + "fmt" + "gorm.io/gorm/callbacks" + + "gorm.io/gorm" +) + +func buildIdentifier(db *gorm.DB) string { + // Build query identifier, + // for that reason we need to compile all arguments into a string + // and concat them with the SQL query itself + + callbacks.BuildQuerySQL(db) + var ( + identifier string + query string + queryArgs string + ) + query = db.Statement.SQL.String() + queryArgs = fmt.Sprintf("%v", db.Statement.Vars) + identifier = fmt.Sprintf("%s-%s", query, queryArgs) + + return identifier +} diff --git a/toolkit/caches/identifier_test.go b/toolkit/caches/identifier_test.go new file mode 100644 index 00000000..68f9c940 --- /dev/null +++ b/toolkit/caches/identifier_test.go @@ -0,0 +1,20 @@ +package caches + +import ( + "testing" + + "gorm.io/gorm" +) + +func Test_buildIdentifier(t *testing.T) { + db := &gorm.DB{} + db.Statement = &gorm.Statement{} + db.Statement.SQL.WriteString("TEST-SQL") + db.Statement.Vars = append(db.Statement.Vars, "test", 123, 12.3, true, false, []string{"test", "me"}) + + actual := buildIdentifier(db) + expected := "TEST-SQL-[test 123 12.3 true false [test me]]" + if actual != expected { + t.Errorf("buildIdentifier expected to return `%s` but got `%s`", expected, actual) + } +} diff --git a/toolkit/caches/query.go b/toolkit/caches/query.go new file mode 100644 index 00000000..adb9ec94 --- /dev/null +++ b/toolkit/caches/query.go @@ -0,0 +1,14 @@ +package caches + +import "gorm.io/gorm" + +type Query struct { + Tags []string + Dest interface{} + RowsAffected int64 +} + +func (q *Query) replaceOn(db *gorm.DB) { + SetPointedValue(db.Statement.Dest, q.Dest) + SetPointedValue(&db.Statement.RowsAffected, &q.RowsAffected) +} diff --git a/toolkit/caches/query_task.go b/toolkit/caches/query_task.go new file mode 100644 index 00000000..799ef77d --- /dev/null +++ b/toolkit/caches/query_task.go @@ -0,0 +1,17 @@ +package caches + +import "gorm.io/gorm" + +type queryTask struct { + id string + db *gorm.DB + queryCb func(db *gorm.DB) +} + +func (q *queryTask) GetId() string { + return q.id +} + +func (q *queryTask) Run() { + q.queryCb(q.db) +} diff --git a/toolkit/caches/query_task_test.go b/toolkit/caches/query_task_test.go new file mode 100644 index 00000000..e721d975 --- /dev/null +++ b/toolkit/caches/query_task_test.go @@ -0,0 +1,38 @@ +package caches + +import ( + "sync/atomic" + "testing" + + "gorm.io/gorm" +) + +func TestQueryTask_GetId(t *testing.T) { + task := &queryTask{ + id: "myId", + db: nil, + queryCb: func(db *gorm.DB) { + }, + } + + if task.GetId() != "myId" { + t.Error("GetId on queryTask returned an unexpected value") + } +} + +func TestQueryTask_Run(t *testing.T) { + var inc int32 + task := &queryTask{ + id: "myId", + db: nil, + queryCb: func(db *gorm.DB) { + atomic.AddInt32(&inc, 1) + }, + } + + task.Run() + + if atomic.LoadInt32(&inc) != 1 { + t.Error("Run on queryTask was expected to execute the callback specified once") + } +} diff --git a/toolkit/caches/reflection.go b/toolkit/caches/reflection.go new file mode 100644 index 00000000..c28a3188 --- /dev/null +++ b/toolkit/caches/reflection.go @@ -0,0 +1,79 @@ +package caches + +import ( + "errors" + "fmt" + "reflect" + + "gorm.io/gorm/schema" +) + +func SetPointedValue(dest interface{}, src interface{}) { + reflect.ValueOf(dest).Elem().Set(reflect.ValueOf(src).Elem()) +} + +func deepCopy(src, dst interface{}) error { + srcVal := reflect.ValueOf(src) + dstVal := reflect.ValueOf(dst) + + if srcVal.Kind() == reflect.Ptr { + srcVal = srcVal.Elem() + } + + if srcVal.Type() != dstVal.Elem().Type() { + return errors.New("src and dst must be of the same type") + } + + return copyValue(srcVal, dstVal.Elem()) +} + +func copyValue(src, dst reflect.Value) error { + switch src.Kind() { + case reflect.Ptr: + src = src.Elem() + dst.Set(reflect.New(src.Type())) + err := copyValue(src, dst.Elem()) + if err != nil { + return err + } + + case reflect.Struct: + for i := 0; i < src.NumField(); i++ { + if src.Type().Field(i).PkgPath != "" { + return fmt.Errorf("%w: %+v", schema.ErrUnsupportedDataType, src.Type().Field(i).Name) + } + err := copyValue(src.Field(i), dst.Field(i)) + if err != nil { + return err + } + } + + case reflect.Slice: + newSlice := reflect.MakeSlice(src.Type(), src.Len(), src.Cap()) + for i := 0; i < src.Len(); i++ { + err := copyValue(src.Index(i), newSlice.Index(i)) + if err != nil { + return err + } + } + dst.Set(newSlice) + + case reflect.Map: + newMap := reflect.MakeMapWithSize(src.Type(), src.Len()) + for _, key := range src.MapKeys() { + value := src.MapIndex(key) + newValue := reflect.New(value.Type()).Elem() + err := copyValue(value, newValue) + if err != nil { + return err + } + newMap.SetMapIndex(key, newValue) + } + dst.Set(newMap) + + default: + dst.Set(src) + } + + return nil +} diff --git a/toolkit/caches/reflection_test.go b/toolkit/caches/reflection_test.go new file mode 100644 index 00000000..7d8683e0 --- /dev/null +++ b/toolkit/caches/reflection_test.go @@ -0,0 +1,215 @@ +package caches + +import ( + "reflect" + "testing" +) + +type unsupportedMockStruct struct { + ExportedField string + unexportedField string + ExportedSliceField []string + unexportedSliceField []string + ExportedMapField map[string]string + unexportedMapField map[string]string +} + +type supportedMockStruct struct { + ExportedField string + ExportedSliceField []string + ExportedMapField map[string]string +} + +func Test_SetPointedValue(t *testing.T) { + src := &struct { + Name string + }{ + Name: "Test", + } + + dest := &struct { + Name string + }{} + + SetPointedValue(dest, src) + + if !reflect.DeepEqual(src, dest) { + t.Error("SetPointedValue was expected to point the dest to the source") + } + + if dest.Name != src.Name { + t.Errorf("src and dest were expected to have the same name, src.Name `%s`, dest.Name `%s`", src.Name, dest.Name) + } +} + +func Test_deepCopy(t *testing.T) { + t.Run("struct", func(t *testing.T) { + t.Run("supported", func(t *testing.T) { + srcStruct := supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := supportedMockStruct{} + + if err := deepCopy(srcStruct, &dstStruct); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + t.Run("unsupported", func(t *testing.T) { + srcStruct := unsupportedMockStruct{ + ExportedField: "exported field", + unexportedField: "unexported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + unexportedSliceField: []string{"1st elem of an unexported slice field", "2nd elem of an unexported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + unexportedMapField: map[string]string{ + "key1": "unexported map elem", + "key2": "unexported map elem", + }, + } + dstStruct := unsupportedMockStruct{} + + if err := deepCopy(srcStruct, &dstStruct); err == nil { + t.Error("deepCopy was expected to fail copying an structure with unexported fields") + } + }) + }) + + t.Run("map", func(t *testing.T) { + t.Run("map[string]string", func(t *testing.T) { + srcMap := map[string]string{ + "key1": "value1", + "key2": "value2", + } + dstMap := make(map[string]string) + + if err := deepCopy(srcMap, &dstMap); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + + t.Run("map[string]struct", func(t *testing.T) { + srcMap := map[string]supportedMockStruct{ + "key1": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + "key2": { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstMap := make(map[string]supportedMockStruct) + + if err := deepCopy(srcMap, &dstMap); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcMap, dstMap) { + t.Errorf("deepCopy failed to copy map: got %+v, want %+v", dstMap, srcMap) + } + }) + }) + + t.Run("slice", func(t *testing.T) { + t.Run("[]string", func(t *testing.T) { + srcSlice := []string{"A", "B", "C"} + dstSlice := make([]string, len(srcSlice)) + + if err := deepCopy(srcSlice, &dstSlice); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + t.Run("[]struct", func(t *testing.T) { + srcSlice := []supportedMockStruct{ + { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, { + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + }, + } + dstSlice := make([]supportedMockStruct, len(srcSlice)) + + if err := deepCopy(srcSlice, &dstSlice); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcSlice, dstSlice) { + t.Errorf("deepCopy failed to copy slice: got %+v, want %+v", dstSlice, srcSlice) + } + }) + }) + + t.Run("pointer", func(t *testing.T) { + srcStruct := &supportedMockStruct{ + ExportedField: "exported field", + ExportedSliceField: []string{"1st elem of an exported slice field", "2nd elem of an exported slice field"}, + ExportedMapField: map[string]string{ + "key1": "exported map elem", + "key2": "exported map elem", + }, + } + dstStruct := &supportedMockStruct{} + + if err := deepCopy(srcStruct, dstStruct); err != nil { + t.Errorf("deepCopy returned an unexpected error %+v", err) + } + + if !reflect.DeepEqual(srcStruct, dstStruct) { + t.Errorf("deepCopy failed to copy structure: got %+v, want %+v", dstStruct, srcStruct) + } + }) + + t.Run("mismatched", func(t *testing.T) { + src := "a string" + dst := 123 + + if err := deepCopy(src, &dst); err == nil { + t.Error("deepCopy did not return an error when provided mismatched types") + } + }) +} diff --git a/toolkit/caches/task.go b/toolkit/caches/task.go new file mode 100644 index 00000000..2b470ff3 --- /dev/null +++ b/toolkit/caches/task.go @@ -0,0 +1,6 @@ +package caches + +type task interface { + GetId() string + Run() +} diff --git a/toolkit/caches/task_test.go b/toolkit/caches/task_test.go new file mode 100644 index 00000000..431a1968 --- /dev/null +++ b/toolkit/caches/task_test.go @@ -0,0 +1,21 @@ +package caches + +import ( + "time" +) + +type mockTask struct { + delay time.Duration + actRes string + expRes string + id string +} + +func (q *mockTask) GetId() string { + return q.id +} + +func (q *mockTask) Run() { + time.Sleep(q.delay) + q.actRes = q.expRes +}