From 53b074af9354fbdd483a1800bfa4b1777dcb7b39 Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Sun, 11 Dec 2022 14:27:27 +0100 Subject: [PATCH] Add internal/geo/location/ --- internal/geo/location/cache.go | 142 +++++++++++++++++++++++++++++++ internal/geo/location/convert.go | 120 ++++++++++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 internal/geo/location/cache.go create mode 100644 internal/geo/location/convert.go diff --git a/internal/geo/location/cache.go b/internal/geo/location/cache.go new file mode 100644 index 0000000..8530c2c --- /dev/null +++ b/internal/geo/location/cache.go @@ -0,0 +1,142 @@ +package location + +import ( + "encoding/json" + "fmt" + "os" + "path" + "strings" + + "github.com/samonzeweb/godb" + "github.com/samonzeweb/godb/adapters/sqlite" + "github.com/zsefvlol/timezonemapper" + + "github.com/chubin/wttr.in/internal/config" + "github.com/chubin/wttr.in/internal/types" +) + +// Cache is an implemenation of DB/file-based cache. +// +// At the moment, it is an implementation for the location cache, +// but it should be generalized to cache everything. +type Cache struct { + config *config.Config + db *godb.DB + indexField string + filesCacheDir string +} + +// NewCache returns new cache reader for the specified config. +func NewCache(config *config.Config) (*Cache, error) { + var ( + db *godb.DB + err error + ) + + if config.Geo.LocationCacheType == types.CacheTypeDB { + db, err = godb.Open(sqlite.Adapter, config.Geo.IPCacheDB) + if err != nil { + return nil, err + } + + // Needed for "upsert" implementation in Put() + db.UseErrorParser() + } + + return &Cache{ + config: config, + db: db, + indexField: "name", + filesCacheDir: config.Geo.LocationCache, + }, nil +} + +// Read returns location information from the cache, if found, +// or types.ErrNotFound if not found. If the entry is found, but its format +// is invalid, types.ErrInvalidCacheEntry is returned. +func (c *Cache) Read(addr string) (*Location, error) { + if c.config.Geo.LocationCacheType == types.CacheTypeFiles { + return c.readFromCacheFile(addr) + } + + return c.readFromCacheDB(addr) +} + +func (c *Cache) readFromCacheFile(name string) (*Location, error) { + var ( + fileLoc = struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Timezone string `json:"timezone"` + Address string `json:"address"` + }{} + location Location + ) + + bytes, err := os.ReadFile(c.cacheFile(name)) + if err != nil { + return nil, types.ErrNotFound + } + err = json.Unmarshal(bytes, &fileLoc) + if err != nil { + return nil, err + } + + // normalize name + name = strings.TrimSpace( + strings.TrimRight( + strings.TrimLeft(name, `"`), `"`)) + + timezone := fileLoc.Timezone + if timezone == "" { + timezone = timezonemapper.LatLngToTimezoneString(fileLoc.Latitude, fileLoc.Longitude) + } + + location = Location{ + Name: name, + Lat: fmt.Sprint(fileLoc.Latitude), + Lon: fmt.Sprint(fileLoc.Longitude), + Timezone: timezone, + Fullname: fileLoc.Address, + } + + return &location, nil +} + +func (c *Cache) readFromCacheDB(addr string) (*Location, error) { + result := Location{} + err := c.db.Select(&result). + Where(c.indexField+" = ?", addr). + Do() + if err != nil { + return nil, err + } + + return &result, nil +} + +func (c *Cache) Put(addr string, loc *Location) error { + if c.config.Geo.IPCacheType == types.CacheTypeDB { + return c.putToCacheDB(loc) + } + + return c.putToCacheFile(addr, loc) +} + +func (c *Cache) putToCacheDB(loc *Location) error { + err := c.db.Insert(loc).Do() + if strings.Contains(fmt.Sprint(err), "UNIQUE constraint failed") { + return c.db.Update(loc).Do() + } + + return err +} + +func (c *Cache) putToCacheFile(addr string, loc fmt.Stringer) error { + return os.WriteFile(c.cacheFile(addr), []byte(loc.String()), 0o600) +} + +// cacheFile returns path to the cache entry for addr. +func (c *Cache) cacheFile(item string) string { + return path.Join(c.filesCacheDir, item) +} diff --git a/internal/geo/location/convert.go b/internal/geo/location/convert.go new file mode 100644 index 0000000..bda2765 --- /dev/null +++ b/internal/geo/location/convert.go @@ -0,0 +1,120 @@ +package location + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + "github.com/samonzeweb/godb" + "github.com/samonzeweb/godb/adapters/sqlite" +) + +//nolint:funlen,cyclop +func (c *Cache) ConvertCache() error { + var ( + dbfile = c.config.Geo.LocationCacheDB + tableName = "Location" + cacheFiles = c.filesCacheDir + known = map[string]bool{} + ) + + err := removeDBIfExists(dbfile) + if err != nil { + return err + } + + db, err := godb.Open(sqlite.Adapter, dbfile) + if err != nil { + return err + } + + err = createTable(db, tableName) + if err != nil { + return err + } + + log.Println("listing cache entries...") + files, err := filepath.Glob(filepath.Join(cacheFiles, "*")) + if err != nil { + return err + } + + log.Printf("going to convert %d entries\n", len(files)) + + block := []Location{} + for i, file := range files { + ip := filepath.Base(file) + loc, err := c.Read(ip) + if err != nil { + log.Println("invalid entry for", ip) + + continue + } + + // Skip duplicates. + if known[loc.Name] { + log.Println("skipping", loc.Name) + + continue + } + known[loc.Name] = true + + // Skip some invalid names. + if strings.Contains(loc.Name, "\n") { + continue + } + + block = append(block, *loc) + if i%1000 != 0 || i == 0 { + continue + } + + log.Println("going to insert new entries") + err = db.BulkInsert(&block).Do() + if err != nil { + return err + } + block = []Location{} + log.Println("converted", i+1, "entries") + } + + // inserting the rest. + err = db.BulkInsert(&block).Do() + if err != nil { + return err + } + + log.Println("converted", len(files), "entries") + + return nil +} + +func createTable(db *godb.DB, tableName string) error { + createTable := fmt.Sprintf( + `create table %s ( + name text not null primary key, + displayName text not null, + lat text not null, + lon text not null, + timezone text not null); + `, tableName) + + _, err := db.CurrentDB().Exec(createTable) + + return err +} + +func removeDBIfExists(filename string) error { + _, err := os.Stat(filename) + if err != nil { + if !os.IsNotExist(err) { + return err + } + // no db file + return nil + } + + return os.Remove(filename) +}