Leak_Technologies/Skyfeed_archive
Archived
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace GNBC WFS with GeoGratis towns endpoint and remove MaxMind

Browse Source
This commit is contained in:
Stu Leak 2025-11-13 21:35:18 -05:00
parent 7f00667213
commit a3e92212d1
7 changed files with 357 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/skyfeed/main.go
View File

@ -16,7 +16,7 @@ var rootCmd = &cobra.Command{
Use: "skyfeed",
Short: "Skyfeed - Open Weather Engine for Telefact and Terminal",
Long: `Skyfeed fetches and normalizes weather data from Environment Canada,
using a local IP database for accurate geolocation. It supports both CLI and API modes.`,
using automatic IP-based geolocation. It supports both CLI and API modes.`,
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
},
@ -29,7 +29,7 @@ func main() {
// Register subcommands
rootCmd.AddCommand(fetchCmd)
rootCmd.AddCommand(showCmd)
rootCmd.AddCommand(updateCmd)
rootCmd.AddCommand(updateTownCmd)
rootCmd.AddCommand(debugLogoCmd)
if err := rootCmd.Execute(); err != nil {
@ -46,13 +46,16 @@ var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Fetch the latest weather data for your current location",
Run: func(cmd *cobra.Command, args []string) {
output.LogInfo("Skyfeed: Checking IP database...")
if err := geo.EnsureIPDBUpToDate(); err != nil {
output.LogError(fmt.Sprintf("Failed to update IP DB: %v", err))
// Ensure town index is present and current
output.LogInfo("Skyfeed: Checking town index…")
if err := geo.EnsureTownIndexUpToDate(); err != nil {
output.LogError(fmt.Sprintf("Failed to update town index: %v", err))
return
}
output.LogInfo("Skyfeed: Detecting location...")
// Detect location via new IP method
output.LogInfo("Skyfeed: Detecting location…")
city, lat, lon, err := geo.GetUserLocation()
if err != nil {
output.LogError(fmt.Sprintf("Could not determine location: %v", err))
@ -60,7 +63,8 @@ var fetchCmd = &cobra.Command{
}
output.LogInfo(fmt.Sprintf("Detected: %s (%.4f, %.4f)", city, lat, lon))
output.LogInfo("Finding nearest Environment Canada station...")
// Find nearest EC station
output.LogInfo("Finding nearest Environment Canada station…")
station, err := geo.FindNearestStation(lat, lon)
if err != nil {
output.LogError(fmt.Sprintf("Station lookup failed: %v", err))
@ -68,7 +72,8 @@ var fetchCmd = &cobra.Command{
}
output.LogInfo(fmt.Sprintf("Nearest station: %s [%s]", station.Name, station.Code))
output.LogInfo("Fetching latest weather data...")
// Fetch weather
output.LogInfo("Fetching latest weather data…")
province := station.Province
data, err := weather.FetchCurrent(station.Code, province)
@ -77,8 +82,8 @@ var fetchCmd = &cobra.Command{
return
}
// Render dynamic header
output.RenderLogo(data.Condition)
// Render dynamic header/logo
output.PrintLogo(data.Condition)
fmt.Println()
// Main output
@ -96,34 +101,34 @@ var showCmd = &cobra.Command{
return
}
output.RenderLogo(data.Condition)
output.PrintLogo(data.Condition)
fmt.Println()
fmt.Println(output.FormatWeatherCLI(data, true))
},
}
var updateCmd = &cobra.Command{
Use: "update-ipdb",
Short: "Manually update the local IP geolocation database",
var updateTownCmd = &cobra.Command{
Use: "update-towns",
Short: "Manually update the Canadian towns geolocation index",
Run: func(cmd *cobra.Command, args []string) {
output.LogInfo("Forcing IP database update...")
if err := geo.ForceUpdateIPDB(); err != nil {
output.LogInfo("Forcing town index update…")
if err := geo.ForceUpdateTownIndex(); err != nil {
output.LogError(fmt.Sprintf("Update failed: %v", err))
return
}
output.LogSuccess("IP database updated successfully.")
output.LogSuccess("Town index updated successfully.")
},
}
// ----------------------------------------------------
// Debug-only utility command
// Debug-only ASCII logo renderer
// ----------------------------------------------------
var debugLogoCmd = &cobra.Command{
Use: "debug-logo [condition]",
Hidden: true,
Short: "Render the dynamic ASCII Skyfeed logo for a given condition (debug use only)",
Short: "Render the dynamic ASCII Skyfeed logo for a given condition",
Long: `This command renders the Skyfeed ASCII logo using a simulated
weather condition. It is intended strictly for development and testing.
Example:
@ -138,6 +143,6 @@ Example:
condition := args[0]
fmt.Printf("Rendering logo for condition: %s\n\n", condition)
output.RenderLogo(condition)
output.PrintLogo(condition)
},
}

121
internal/geo/geolocate.go
View File

@ -5,15 +5,16 @@ import (
"fmt"
"net"
"net/http"
"path/filepath"
"strings"
"time"
"git.leaktechnologies.dev/Leak_Technologies/Skyfeed/internal/config"
"github.com/oschwald/geoip2-golang"
)
// GetUserLocation resolves the user's public IP into a city and coordinates.
// It will try multiple fallback IP providers if the first one fails.
// -------------------------------
// Public IP + Geolocation (ipwho.is)
// -------------------------------
// GetUserLocation resolves the user's IP into:
// city, latitude, longitude
func GetUserLocation() (string, float64, float64, error) {
fmt.Println("[geo] Detecting location...")
@ -22,36 +23,22 @@ func GetUserLocation() (string, float64, float64, error) {
return "", 0, 0, fmt.Errorf("failed to resolve public IP: %w", err)
}
dbPath := filepath.Join(config.DataDir, "GeoLite2-City.mmdb")
db, err := geoip2.Open(dbPath)
city, province, lat, lon, err := lookupIP(ip.String())
if err != nil {
return "", 0, 0, fmt.Errorf("failed to open GeoLite2 database: %w", err)
}
defer db.Close()
record, err := db.City(ip)
if err != nil {
return "", 0, 0, fmt.Errorf("geoip lookup failed: %w", err)
return "", 0, 0, fmt.Errorf("IP geolocation failed: %w", err)
}
city := record.City.Names["en"]
prov := ""
if len(record.Subdivisions) > 0 {
prov = record.Subdivisions[0].Names["en"]
// e.g., "Ottawa, Ontario"
locationName := fmt.Sprintf("%s, %s", city, province)
fmt.Printf("[geo] Detected: %s (%.4f, %.4f)\n", locationName, lat, lon)
return locationName, lat, lon, nil
}
lat := record.Location.Latitude
lon := record.Location.Longitude
// -------------------------------
// PUBLIC IP DETECTION
// -------------------------------
if city == "" && prov == "" {
return "", 0, 0, fmt.Errorf("no location info found for IP %s", ip.String())
}
fmt.Printf("[geo] Detected: %s, %s (%.4f, %.4f)\n", city, prov, lat, lon)
return fmt.Sprintf("%s, %s", city, prov), lat, lon, nil
}
// fetchPublicIP tries multiple reliable endpoints for the public IPv4 address.
func fetchPublicIP() (net.IP, error) {
providers := []string{
"https://ipv4.icanhazip.com",
@ -72,11 +59,10 @@ func fetchPublicIP() (net.IP, error) {
return nil, fmt.Errorf("all IP detection methods failed")
}
// tryProvider queries a single IP API endpoint and parses IPv4 results.
func tryProvider(url string, client *http.Client) (net.IP, error) {
resp, err := client.Get(url)
if err != nil {
return nil, fmt.Errorf("network error (%s): %w", url, err)
return nil, err
}
defer resp.Body.Close()
@ -84,30 +70,79 @@ func tryProvider(url string, client *http.Client) (net.IP, error) {
return nil, fmt.Errorf("HTTP %d (%s)", resp.StatusCode, url)
}
// Some APIs return plain text, others JSON
var result struct {
IP string `json:"ip"`
}
// JSON-based API fallback
var j struct{ IP string `json:"ip"` }
// Try decode JSON
if err := json.NewDecoder(resp.Body).Decode(&result); err == nil && result.IP != "" {
ip := net.ParseIP(result.IP)
if err := json.NewDecoder(resp.Body).Decode(&j); err == nil && j.IP != "" {
ip := net.ParseIP(j.IP)
if ip != nil && ip.To4() != nil {
return ip, nil
}
}
// Fallback: plain text
// Plain text fallback → must re-fetch body
resp2, err := client.Get(url)
if err == nil {
if err != nil {
return nil, err
}
defer resp2.Body.Close()
buf := make([]byte, 64)
n, _ := resp2.Body.Read(buf)
ip := net.ParseIP(string(buf[:n]))
ip := net.ParseIP(strings.TrimSpace(string(buf[:n])))
if ip != nil && ip.To4() != nil {
return ip, nil
}
return nil, fmt.Errorf("invalid response from %s", url)
}
return nil, fmt.Errorf("no valid IP found from %s", url)
// -------------------------------
// GEO LOOKUP USING ipwho.is
// -------------------------------
func lookupIP(ip string) (city string, province string, lat float64, lon float64, err error) {
url := fmt.Sprintf("https://ipwho.is/%s", ip)
client := &http.Client{Timeout: 6 * time.Second}
resp, err := client.Get(url)
if err != nil {
return "", "", 0, 0, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", "", 0, 0, fmt.Errorf("ipwho.is returned HTTP %d", resp.StatusCode)
}
var data struct {
Success bool `json:"success"`
City string `json:"city"`
Region string `json:"region"`
Lat float64 `json:"latitude"`
Lon float64 `json:"longitude"`
Message string `json:"message"`
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return "", "", 0, 0, fmt.Errorf("decode error: %w", err)
}
if !data.Success {
return "", "", 0, 0, fmt.Errorf("ipwho.is error: %s", data.Message)
}
// Clean fields
city = strings.TrimSpace(data.City)
province = strings.TrimSpace(data.Region)
if city == "" {
city = "Unknown"
}
if province == "" {
province = "Unknown"
}
return city, province, data.Lat, data.Lon, nil
}

137
internal/geo/ipdb_updater.go
View File

@ -1,137 +0,0 @@
package geo
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"git.leaktechnologies.dev/Leak_Technologies/Skyfeed/internal/config"
)
const (
ipdbFileName = "GeoLite2-City.mmdb"
keyFileName = "maxmind.key"
)
// EnsureIPDBUpToDate checks the local MaxMind database and refreshes monthly.
func EnsureIPDBUpToDate() error {
dbPath := filepath.Join(config.DataDir, ipdbFileName)
info, err := os.Stat(dbPath)
if os.IsNotExist(err) {
fmt.Println("[geo] No IP database found, downloading...")
return updateIPDB(dbPath)
}
if err != nil {
return fmt.Errorf("unable to check IP DB: %w", err)
}
modTime := info.ModTime().UTC()
now := time.Now().UTC()
firstOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
if modTime.Before(firstOfMonth) {
fmt.Println("[geo] IP database is older than this month, refreshing...")
return updateIPDB(dbPath)
}
fmt.Println("[geo] IP database is current.")
return nil
}
// ForceUpdateIPDB forces an immediate refresh.
func ForceUpdateIPDB() error {
dbPath := filepath.Join(config.DataDir, ipdbFileName)
fmt.Println("[geo] Forcing IP database update...")
return updateIPDB(dbPath)
}
// updateIPDB downloads and extracts the official GeoLite2 City database using your MaxMind key.
func updateIPDB(dest string) error {
keyPath := filepath.Join(config.ConfigDir, keyFileName)
keyBytes, err := os.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("[geo] Missing MaxMind license key.\nPlease run:\n echo \"YOUR_KEY\" > %s", keyPath)
}
key := strings.TrimSpace(string(keyBytes))
url := fmt.Sprintf("https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=%s&suffix=tar.gz", key)
tmpTar := dest + ".tar.gz"
if err := downloadFile(url, tmpTar); err != nil {
return fmt.Errorf("failed to download GeoLite2 archive: %w", err)
}
defer os.Remove(tmpTar)
if err := extractMMDB(tmpTar, dest); err != nil {
return fmt.Errorf("failed to extract mmdb: %w", err)
}
fmt.Println("[geo] IP database updated successfully →", dest)
return nil
}
// downloadFile streams a file from URL to disk.
func downloadFile(url, dest string) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected HTTP status: %s", resp.Status)
}
out, err := os.Create(dest)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
}
// extractMMDB extracts the .mmdb file from a tar.gz archive.
func extractMMDB(src, dest string) error {
f, err := os.Open(src)
if err != nil {
return err
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
return err
}
defer gz.Close()
tr := tar.NewReader(gz)
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if filepath.Ext(h.Name) == ".mmdb" {
out, err := os.Create(dest)
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, tr)
return err
}
}
return fmt.Errorf("no .mmdb found in archive")
}

10
internal/geo/towns.go Normal file
View File

@ -0,0 +1,10 @@
package geo
// Town represents a Canadian populated place.
// Shared by both the updater and lookup system.
type Town struct {
Name string `json:"name"`
Province string `json:"province"`
Lat float64 `json:"lat"`
Lon float64 `json:"lon"`
}

14
internal/geo/towns_lookup.go
View File

@ -10,14 +10,6 @@ import (
"git.leaktechnologies.dev/Leak_Technologies/Skyfeed/internal/config"
)
// Town represents a Canadian town record (loaded from towns.json).
type Town struct {
Name string `json:"name"`
Province string `json:"province"`
Lat float64 `json:"lat"`
Lon float64 `json:"lon"`
}
// FindNearestTown loads the cached towns.json and finds the closest town to given coordinates.
func FindNearestTown(lat, lon float64) (Town, error) {
townsPath := filepath.Join(config.DataDir, "towns.json")
@ -40,7 +32,7 @@ func FindNearestTown(lat, lon float64) (Town, error) {
var nearest Town
for _, t := range towns {
d := Haversine(lat, lon, t.Lat, t.Lon) // ✅ use shared helper from stations.go
d := Haversine(lat, lon, t.Lat, t.Lon)
if d < minDist {
minDist = d
nearest = t
@ -51,6 +43,8 @@ func FindNearestTown(lat, lon float64) (Town, error) {
return Town{}, fmt.Errorf("no nearby town found")
}
fmt.Printf("[geo] Nearest town: %s, %s (%.2f km)\n", nearest.Name, nearest.Province, minDist)
fmt.Printf("[geo] Nearest town: %s, %s (%.2f km)\n",
nearest.Name, nearest.Province, minDist)
return nearest, nil
}

54
internal/geo/towns_updater.go
View File

@ -13,21 +13,12 @@ import (
)
const (
// Official Geographical Names Board of Canada WFS API
// Docs: https://www.nrcan.gc.ca/earth-sciences/geography/geographical-names-board-canada/download-geographical-names-data/10786
gnbcAPIURL = "https://geogratis.gc.ca/geonames/servlet/com.gc.ccra.geonames.webservices.GeographicalNamesService?service=WFS&request=GetFeature&version=2.0.0&typeNames=geonames:geoname_eng&outputFormat=json&featureCode=PPL"
// WORKING GeoGratis endpoint for all populated places (PPL)
gnbcAPIURL = "https://geogratis.gc.ca/services/geoname/en/geonames.json?feature_code=PPL"
townsFile = "towns.json"
maxFetchTime = 5 * time.Minute
)
// TownRecord represents a single Canadian town.
type TownRecord struct {
Name string `json:"name"`
Province string `json:"province"`
Lat float64 `json:"lat"`
Lon float64 `json:"lon"`
}
// EnsureTownIndexUpToDate checks if the towns index needs updating (monthly).
func EnsureTownIndexUpToDate() error {
dest := filepath.Join(config.DataDir, townsFile)
@ -62,11 +53,10 @@ func ForceUpdateTownIndex() error {
return downloadTownIndex(dest)
}
// downloadTownIndex fetches and stores the Canadian town dataset.
func downloadTownIndex(dest string) error {
client := &http.Client{Timeout: maxFetchTime}
fmt.Println("[geo] Fetching town data from GNBC WFS API...")
fmt.Println("[geo] Fetching town data from GeoGratis...")
resp, err := client.Get(gnbcAPIURL)
if err != nil {
return fmt.Errorf("failed to fetch town dataset: %w", err)
@ -79,17 +69,17 @@ func downloadTownIndex(dest string) error {
raw, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed to read GNBC response: %w", err)
return fmt.Errorf("failed to read GeoGratis response: %w", err)
}
towns, err := parseGNBCJSON(raw)
if err != nil {
return fmt.Errorf("failed to parse GNBC JSON: %w", err)
return fmt.Errorf("failed to parse GeoGratis JSON: %w", err)
}
data, err := json.MarshalIndent(towns, "", " ")
if err != nil {
return fmt.Errorf("failed to encode towns: %w", err)
return fmt.Errorf("failed to encode towns.json: %w", err)
}
if err := os.WriteFile(dest, data, 0644); err != nil {
@ -100,34 +90,32 @@ func downloadTownIndex(dest string) error {
return nil
}
// parseGNBCJSON extracts relevant town info from the GNBC GeoJSON.
func parseGNBCJSON(data []byte) ([]TownRecord, error) {
// Uses GeoGratis geonames.json structure
func parseGNBCJSON(data []byte) ([]Town, error) {
var response struct {
Features []struct {
Properties struct {
Items []struct {
Name string `json:"name"`
Province string `json:"province"`
ProvinceCode string `json:"province_code"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
} `json:"properties"`
} `json:"features"`
} `json:"items"`
}
if err := json.Unmarshal(data, &response); err != nil {
return nil, fmt.Errorf("invalid GNBC JSON: %w", err)
return nil, fmt.Errorf("invalid GeoGratis JSON: %w", err)
}
var towns []TownRecord
for _, f := range response.Features {
p := f.Properties
if p.Name == "" || p.Province == "" {
var towns []Town
for _, item := range response.Items {
if item.Name == "" || item.ProvinceCode == "" {
continue
}
towns = append(towns, TownRecord{
Name: p.Name,
Province: p.Province,
Lat: p.Latitude,
Lon: p.Longitude,
towns = append(towns, Town{
Name: item.Name,
Province: item.ProvinceCode,
Lat: item.Latitude,
Lon: item.Longitude,
})
}

BIN
skyfeed
View File

Binary file not shown.