Add geoloc cache converter

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/smartystreets/assertions v1.2.0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect
+	github.com/zsefvlol/timezonemapper v1.0.0 // indirect
 	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -38,6 +38,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/zsefvlol/timezonemapper v1.0.0 h1:HXqkOzf01gXYh2nDQcDSROikFgMaximnhE8BY9SyF6E=
+github.com/zsefvlol/timezonemapper v1.0.0/go.mod h1:cVUCOLEmc/VvOMusEhpd2G/UBtadL26ZVz2syODXDoQ=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

internal/config/config.go

@@ -77,7 +77,15 @@ type Geo struct {
 	// IPCacheDB contains the path to the SQLite DB with the IP Geodata cache.
 	IPCacheDB string `yaml:"ipCacheDB,omitempty"`
 
-	CacheType types.CacheType `yaml:"cacheType,omitempty"`
+	IPCacheType types.CacheType `yaml:"cacheType,omitempty"`
+
+	// LocationCache contains the path to the Location Geodata cache.
+	LocationCache string `yaml:"locationCache,omitempty"`
+
+	// LocationCacheDB contains the path to the SQLite DB with the Location Geodata cache.
+	LocationCacheDB string `yaml:"locationCacheDB,omitempty"`
+
+	LocationCacheType types.CacheType `yaml:"locationCacheType,omitempty"`
 
 	Nominatim []Nominatim
 }
@@ -97,9 +105,12 @@ func Default() *Config {
 			Size: 12800,
 		},
 		Geo{
-			IPCache:   "/wttr.in/cache/ip2l",
-			IPCacheDB: "/wttr.in/cache/geoip.db",
-			CacheType: types.CacheTypeDB,
+			IPCache:           "/wttr.in/cache/ip2l",
+			IPCacheDB:         "/wttr.in/cache/geoip.db",
+			IPCacheType:       types.CacheTypeDB,
+			LocationCache:     "/wttr.in/cache/loc",
+			LocationCacheDB:   "/wttr.in/cache/geoloc.db",
+			LocationCacheType: types.CacheTypeFiles,
 			Nominatim: []Nominatim{
 				{
 					Name:  "locationiq",

internal/geo/ip/convert.go

@@ -3,17 +3,18 @@ package ip
 import (
 	"fmt"
 	"log"
-	"os"
 	"path/filepath"
 
 	"github.com/samonzeweb/godb"
 	"github.com/samonzeweb/godb/adapters/sqlite"
+
+	"github.com/chubin/wttr.in/internal/util"
 )
 
 func (c *Cache) ConvertCache() error {
 	dbfile := c.config.Geo.IPCacheDB
 
-	err := removeDBIfExists(dbfile)
+	err := util.RemoveFileIfExists(dbfile)
 	if err != nil {
 		return err
 	}
@@ -29,7 +30,7 @@ func (c *Cache) ConvertCache() error {
 	}
 
 	log.Println("listing cache entries...")
-	files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*"))
+	files, err := filepath.Glob(filepath.Join(c.config.Geo.LocationCache, "*"))
 	if err != nil {
 		return err
 	}
@@ -72,28 +73,12 @@ func (c *Cache) ConvertCache() error {
 func createTable(db *godb.DB, tableName string) error {
 	createTable := fmt.Sprintf(
 		`create table %s (
-	    ip             text not null primary key,
-        countryCode    text not null,
-        country        text not null,
-        region         text not null,
-        city           text not null,
-        latitude       text not null,
-        longitude      text not null);
+	    name           text not null primary key,
+        fullName       text not null,
+        lat            text not null,
+        long           text not null);
 	`, tableName)
 
 	_, err := db.CurrentDB().Exec(createTable)
 	return err
 }
-
-func removeDBIfExists(filename string) error {
-	_, err := os.Stat(filename)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		// no db file
-		return nil
-	}
-
-	return os.Remove(filename)
-}

internal/geo/ip/ip.go

@@ -1,7 +1,6 @@
 package ip
 
 import (
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -19,11 +18,6 @@ import (
 	"github.com/samonzeweb/godb/adapters/sqlite"
 )
 
-var (
-	ErrNotFound          = errors.New("cache entry not found")
-	ErrInvalidCacheEntry = errors.New("invalid cache entry format")
-)
-
 // Location information.
 type Location struct {
 	IP          string  `db:"ip,key"`
@@ -69,8 +63,8 @@ func NewCache(config *config.Config) (*Cache, error) {
 }
 
 // Read returns location information from the cache, if found,
-// or ErrNotFound if not found. If the entry is found, but its format
-// is invalid, ErrInvalidCacheEntry is returned.
+// or types.ErrNotFound if not found. If the entry is found, but its format
+// is invalid, types.ErrInvalidCacheEntry is returned.
 //
 // Format:
 //
@@ -81,7 +75,7 @@ func NewCache(config *config.Config) (*Cache, error) {
 //     DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782
 //
 func (c *Cache) Read(addr string) (*Location, error) {
-	if c.config.Geo.CacheType == types.CacheTypeDB {
+	if c.config.Geo.IPCacheType == types.CacheTypeDB {
 		return c.readFromCacheDB(addr)
 	}
 	return c.readFromCacheFile(addr)
@@ -90,7 +84,7 @@ func (c *Cache) Read(addr string) (*Location, error) {
 func (c *Cache) readFromCacheFile(addr string) (*Location, error) {
 	bytes, err := os.ReadFile(c.cacheFile(addr))
 	if err != nil {
-		return nil, ErrNotFound
+		return nil, types.ErrNotFound
 	}
 	return NewLocationFromString(addr, string(bytes))
 }
@@ -107,7 +101,7 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) {
 }
 
 func (c *Cache) Put(addr string, loc *Location) error {
-	if c.config.Geo.CacheType == types.CacheTypeDB {
+	if c.config.Geo.IPCacheType == types.CacheTypeDB {
 		return c.putToCacheDB(addr, loc)
 	}
 	return c.putToCacheFile(addr, loc)
@@ -150,18 +144,18 @@ func NewLocationFromString(addr, s string) (*Location, error) {
 
 	parts := strings.Split(s, ";")
 	if len(parts) < 4 {
-		return nil, ErrInvalidCacheEntry
+		return nil, types.ErrInvalidCacheEntry
 	}
 
 	if len(parts) >= 6 {
 		lat, err = strconv.ParseFloat(parts[4], 64)
 		if err != nil {
-			return nil, ErrInvalidCacheEntry
+			return nil, types.ErrInvalidCacheEntry
 		}
 
 		long, err = strconv.ParseFloat(parts[5], 64)
 		if err != nil {
-			return nil, ErrInvalidCacheEntry
+			return nil, types.ErrInvalidCacheEntry
 		}
 	}
 

internal/geo/ip/ip_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/chubin/wttr.in/internal/types"
 )
 
 func TestParseCacheEntry(t *testing.T) {
@@ -63,7 +65,7 @@ func TestParseCacheEntry(t *testing.T) {
 			"1.2.3.4",
 			"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX",
 			Location{},
-			ErrInvalidCacheEntry,
+			types.ErrInvalidCacheEntry,
 		},
 	}
 

internal/geo/location/location.go

@@ -1,12 +1,29 @@
 package location
 
-import "github.com/chubin/wttr.in/internal/config"
+import (
+	"encoding/json"
+	"log"
+
+	"github.com/chubin/wttr.in/internal/config"
+)
 
 type Location struct {
-	Name     string
-	Fullname string `json:"display_name"`
-	Lat      string
-	Lon      string
+	Name     string `db:"name,key"`
+	Fullname string `db:"displayName" json:"display_name"`
+	Lat      string `db:"lat"`
+	Lon      string `db:"lon"`
+	Timezone string `db:"timezone"`
+}
+
+// String returns string represenation of location
+func (l *Location) String() string {
+	bytes, err := json.Marshal(l)
+	if err != nil {
+		// should never happen
+		log.Fatalln(err)
+	}
+
+	return string(bytes)
 }
 
 type Provider interface {

internal/types/types.go

@@ -1,8 +1,15 @@
 package types
 
+import "errors"
+
 type CacheType string
 
 const (
 	CacheTypeDB    = "db"
 	CacheTypeFiles = "files"
 )
+
+var (
+	ErrNotFound          = errors.New("cache entry not found")
+	ErrInvalidCacheEntry = errors.New("invalid cache entry format")
+)

srv.go

@@ -25,7 +25,8 @@ var cli struct {
 
 	ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"`
 
-	ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
+	ConvertGeoIPCache       bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
+	ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"`
 }
 
 const logLineStart = "LOG_LINE_START "
@@ -188,6 +189,15 @@ func main() {
 		return
 	}
 
+	if cli.ConvertGeoLocationCache {
+		geoLocCache, err := geoloc.NewCache(conf)
+		if err != nil {
+			ctx.FatalIfErrorf(err)
+		}
+		ctx.FatalIfErrorf(geoLocCache.ConvertCache())
+		return
+	}
+
 	if cli.GeoResolve != "" {
 		sr := geoloc.NewSearcher(conf)
 		loc, err := sr.Search(cli.GeoResolve)