Merge pull request 'Optimize lookup speed / add net address cache' (#13) from optimize_lookup_speed into main

Reviewed-on: #13

api/v1/server.go

@@ -3,6 +3,7 @@ package api_v1
 import (
 	"context"
 	"net/http"
+	"time"
 
 	"git.ar21.de/yolokube/country-geo-locations/pkg/geoloc"
 	"github.com/go-chi/chi/v5"
@@ -49,16 +50,20 @@ func searchIPCtx(next http.Handler) http.Handler {
 				render.Render(w, r, errInvalidRequest(err))
 				return
 			}
-			for {
-				ipinfo, err = geoloc.SearchIPNet(ipnetnum)
-				if err != nil {
-					render.Render(w, r, errNotFound)
-					return
-				}
-				if ipinfo != nil {
-					break
-				}
-				ipnetnum = ipnetnum - 8
+
+			newipnet, found := geoloc.IPCache.Get(ipnetnum)
+			if found {
+				ipnetnum = newipnet
+			}
+
+			ipinfo, err = geoloc.SearchIPNet(ipnetnum)
+			if err != nil {
+				render.Render(w, r, errNotFound)
+				return
+			}
+
+			if !found {
+				geoloc.IPCache.Set(ipnetnum, ipinfo.IPNumFrom, 2*time.Minute)
 			}
 		}
 		ctx := context.WithValue(r.Context(), keyIPInfo, ipinfo)

go.sum

@@ -9,8 +9,6 @@ github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygv
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-chi/chi/v5 v5.0.11 h1:BnpYbFZ3T3S1WMpD79r7R5ThWX40TaFB7L31Y8xqSwA=
-github.com/go-chi/chi/v5 v5.0.11/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
 github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
@@ -42,12 +40,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/praserx/ipconv v1.2.1 h1:MWGfrF+OZ0pqIuTlNlMgvJDDbohC3h751oN1+Ov3x4k=
 github.com/praserx/ipconv v1.2.1/go.mod h1:DSy+AKre/e3w/npsmUDMio+OR/a2rvmMdI7rerOIgqI=
-github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
-github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/schollz/progressbar/v3 v3.14.1 h1:VD+MJPCr4s3wdhTc7OEJ/Z3dAeBzJ7yKH/P4lC5yRTI=
-github.com/schollz/progressbar/v3 v3.14.1/go.mod h1:Zc9xXneTzWXF81TGoqL71u0sBPjULtEHYtj/WVgVy8E=
 github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
 github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -56,14 +50,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
 golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
-golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE=
-golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
 golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -32,6 +32,9 @@ func main() {
 	}
 	fmt.Println("Import data from file successful")
 
+	geoloc.IPCache = geoloc.NewCache()
+	fmt.Println("Cache created")
+
 	r := chi.NewRouter()
 	r.Use(middleware.RequestID)
 	r.Use(middleware.Logger)

pkg/geoloc/cache.go

@@ -0,0 +1,29 @@
+package geoloc
+
+import (
+	"sync"
+	"time"
+)
+
+var IPCache *Cache
+
+type Cache struct {
+	store sync.Map
+}
+
+func NewCache() *Cache {
+	return &Cache{}
+}
+
+func (c *Cache) Set(key, value uint, ttl time.Duration) {
+	c.store.Store(key, value)
+	time.AfterFunc(ttl, func() {
+		c.store.Delete(key)
+	})
+}
+
+func (c *Cache) Get(key uint) (uint, bool) {
+	output, _ := c.store.Load(key)
+	value, ok := output.(uint)
+	return value, ok
+}

pkg/geoloc/database.go

@@ -8,7 +8,7 @@ import (
 
 var database *memdb.MemDB
 
-func CreateDatabase(ipinfos []IPInfo) error {
+func CreateDatabase(ipinfos []IPInfo) (err error) {
 	schema := &memdb.DBSchema{
 		Tables: map[string]*memdb.TableSchema{
 			"ipinfo": {
@@ -24,38 +24,28 @@ func CreateDatabase(ipinfos []IPInfo) error {
 		},
 	}
 
-	db, err := memdb.NewMemDB(schema)
+	database, err = memdb.NewMemDB(schema)
 	if err != nil {
-		return err
+		return
 	}
 
-	txn := db.Txn(true)
+	txn := database.Txn(true)
 	defer txn.Abort()
 
 	for _, ipinfo := range ipinfos {
-		if err := txn.Insert("ipinfo", ipinfo); err != nil {
-			return err
+		if err = txn.Insert("ipinfo", ipinfo); err != nil {
+			return
 		}
 	}
 
 	txn.Commit()
-
-	database = db
-	return nil
+	return
 }
 
 func SearchIPNet(ipnetnum uint) (*IPInfo, error) {
 	txn := database.Txn(false)
 	defer txn.Abort()
 
-	raw, err := txn.First("ipinfo", "id", ipnetnum)
-	if err != nil {
-		return nil, err
-	}
-	if raw == nil {
-		return nil, nil
-	}
-
 	var ipinfo IPInfo
 	for {
 		raw, err := txn.First("ipinfo", "id", ipnetnum)