Mirrors/wttr.in
2
0
Fork 0
mirror of https://github.com/chubin/wttr.in synced 2024-11-14 16:17:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

Add geoip cache converter

Browse source
This commit is contained in:
Igor Chubin 2022-12-04 21:16:23 +01:00
parent bcb3667aef
commit b54ba36643
5 changed files with 132 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
internal/config/config.go
View file

@ -72,6 +72,9 @@ type Cache struct {
type Geo struct {
// IPCache contains the path to the IP Geodata cache.
IPCache string `yaml:"ipCache,omitempty"`
// IPCacheDB contains the path to the SQLite DB with the IP Geodata cache.
IPCacheDB string `yaml:"ipCacheDB,omitempty"`
}
// Default contains the default configuration.
@ -81,7 +84,8 @@ func Default() *Config {
Size: 12800,
},
Geo{
IPCache: "/wttr.in/cache/ip2l",
IPCache: "/wttr.in/cache/ip2l",
IPCacheDB: "/wttr.in/cache/geoip.db",
},
Logging{
AccessLog: "/wttr.in/log/access.log",

99
internal/geo/ip/convert.go Normal file
View file

@ -0,0 +1,99 @@
package ip
import (
"fmt"
"log"
"os"
"path/filepath"
"github.com/samonzeweb/godb"
"github.com/samonzeweb/godb/adapters/sqlite"
)
func (c *Cache) ConvertCache() error {
dbfile := c.config.Geo.IPCacheDB
err := removeDBIfExists(dbfile)
if err != nil {
return err
}
db, err := godb.Open(sqlite.Adapter, dbfile)
if err != nil {
return err
}
err = createTable(db, "Location")
if err != nil {
return err
}
log.Println("listing cache entries...")
files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*"))
if err != nil {
return err
}
log.Printf("going to convert %d entries\n", len(files))
block := []Location{}
for i, file := range files {
ip := filepath.Base(file)
loc, err := c.Read(ip)
if err != nil {
log.Println("invalid entry for", ip)
continue
}
block = append(block, *loc)
if i%1000 != 0 || i == 0 {
continue
}
err = db.BulkInsert(&block).Do()
if err != nil {
return err
}
block = []Location{}
log.Println("converted", i+1, "entries")
}
// inserting the rest.
err = db.BulkInsert(&block).Do()
if err != nil {
return err
}
log.Println("converted", len(files), "entries")
return nil
}
func createTable(db *godb.DB, tableName string) error {
createTable := fmt.Sprintf(
`create table %s (
ip text not null primary key,
countryCode text not null,
country text not null,
region text not null,
city text not null,
latitude text not null,
longitude text not null);
`, tableName)
_, err := db.CurrentDB().Exec(createTable)
return err
}
func removeDBIfExists(filename string) error {
_, err := os.Stat(filename)
if err != nil {
if !os.IsNotExist(err) {
return err
}
// no db file
return nil
}
return os.Remove(filename)
}

18
internal/geo/ip/ip.go
View file

@ -22,12 +22,13 @@ var (
// Location information.
type Location struct {
CountryCode string
Country string
Region string
City string
Latitude float64
Longitude float64
IP string `db:"ip,key"`
CountryCode string `db:"countryCode"`
Country string `db:"country"`
Region string `db:"region"`
City string `db:"city"`
Latitude float64 `db:"latitude"`
Longitude float64 `db:"longitude"`
}
// Cache provides access to the IP Geodata cache.
@ -59,7 +60,7 @@ func (c *Cache) Read(addr string) (*Location, error) {
if err != nil {
return nil, ErrNotFound
}
return parseCacheEntry(string(bytes))
return parseCacheEntry(addr, string(bytes))
}
// cacheFile retuns path to the cache entry for addr.
@ -69,7 +70,7 @@ func (c *Cache) cacheFile(addr string) string {
// parseCacheEntry parses the location cache entry s,
// and return location, or error, if the cache entry is invalid.
func parseCacheEntry(s string) (*Location, error) {
func parseCacheEntry(addr, s string) (*Location, error) {
var (
lat float64 = -1000
long float64 = -1000
@ -94,6 +95,7 @@ func parseCacheEntry(s string) (*Location, error) {
}
return &Location{
IP: addr,
CountryCode: parts[0],
Country: parts[1],
Region: parts[2],

10
internal/geo/ip/ip_test.go
View file

@ -8,13 +8,16 @@ import (
func TestParseCacheEntry(t *testing.T) {
tests := []struct {
addr string
input string
expected Location
err error
}{
{
"1.2.3.4",
"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782",
Location{
IP: "1.2.3.4",
CountryCode: "DE",
Country: "Germany",
Region: "Free and Hanseatic City of Hamburg",
@ -26,8 +29,10 @@ func TestParseCacheEntry(t *testing.T) {
},
{
"1.2.3.4",
"ES;Spain;Madrid, Comunidad de;Madrid;40.4165;-3.70256;28223;Orange Espagne SA;orange.es",
Location{
IP: "1.2.3.4",
CountryCode: "ES",
Country: "Spain",
Region: "Madrid, Comunidad de",
@ -39,8 +44,10 @@ func TestParseCacheEntry(t *testing.T) {
},
{
"1.2.3.4",
"US;United States of America;California;Mountain View",
Location{
IP: "1.2.3.4",
CountryCode: "US",
Country: "United States of America",
Region: "California",
@ -53,6 +60,7 @@ func TestParseCacheEntry(t *testing.T) {
// Invalid entries
{
"1.2.3.4",
"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX",
Location{},
ErrInvalidCacheEntry,
@ -60,7 +68,7 @@ func TestParseCacheEntry(t *testing.T) {
}
for _, tt := range tests {
result, err := parseCacheEntry(tt.input)
result, err := parseCacheEntry(tt.addr, tt.input)
if tt.err == nil {
require.NoError(t, err)
require.Equal(t, *result, tt.expected)

9
srv.go
View file

@ -12,6 +12,7 @@ import (
"github.com/alecthomas/kong"
"github.com/chubin/wttr.in/internal/config"
geoip "github.com/chubin/wttr.in/internal/geo/ip"
"github.com/chubin/wttr.in/internal/logging"
"github.com/chubin/wttr.in/internal/processor"
)
@ -20,6 +21,8 @@ var cli struct {
ConfigCheck bool `name:"config-check" help:"Check configuration"`
ConfigDump bool `name:"config-dump" help:"Dump configuration"`
ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"`
ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
}
const logLineStart = "LOG_LINE_START "
@ -173,6 +176,12 @@ func main() {
return
}
if cli.ConvertGeoIPCache {
geoIPCache := geoip.NewCache(conf)
ctx.FatalIfErrorf(geoIPCache.ConvertCache())
return
}
err = serve(conf)
ctx.FatalIfErrorf(err)
}