Goondex/cmd/goondex/main.go
Team Goon f7d82cd207 v0.1.0-dev2: Full TPDB integration with auto-fetch and comprehensive docs
Major Features:
-  Complete TPDB scraper implementation with real API calls
-  Auto-fetch on cache miss: search commands now automatically import from TPDB when not found locally
-  Comprehensive documentation (5 markdown files for Bookstack)
-  Import commands for performers, studios, and scenes
-  Fixed JSON type mismatches (aliases array, studio numeric IDs)

Changes:
1. TPDB Scraper (internal/scraper/tpdb/):
   - types.go: Full API response structures with correct types
     - PerformerResponse.Aliases: string → []string (TPDB returns array)
     - StudioResponse.ID: string → int (TPDB returns numeric IDs)
     - SiteInfo.ID: string → int (scenes reference studios by number)
   - mapper.go: Maps TPDB responses to internal models
     - Converts aliases array to comma-separated string
     - Converts numeric studio IDs to strings using strconv.Itoa()
   - scraper.go: Real HTTP client with Bearer token auth
     - SearchPerformers, SearchStudios, SearchScenes implemented
     - GetPerformerByID, GetStudioByID, GetSceneByID implemented

2. CLI Auto-Fetch (cmd/goondex/main.go):
   - performer-search: Auto-fetches from TPDB if local DB empty
   - studio-search: Auto-fetches from TPDB if local DB empty
   - scene-search: Auto-fetches basic metadata (no relationships)
   - Graceful handling of missing TPDB_API_KEY
   - Import → search again to get local IDs

3. Documentation (docs/):
   - INDEX.md: Documentation overview and navigation
   - ARCHITECTURE.md: System design, data flow, component diagrams
   - DATABASE_SCHEMA.md: Complete schema with relationships and indexes
   - CLI_REFERENCE.md: All commands with examples
   - TPDB_INTEGRATION.md: API guide, data mapping, best practices

4. Fixes:
   - .gitignore: Fixed pattern to allow cmd/goondex/* and cmd/goondexd/*
   - README: Updated to reflect TPDB integration and auto-fetch

Testing:
-  performer-search "Riley Reid" - auto-fetched 2 performers, cached
-  studio-search "Brazzers" - auto-fetched 12 studios, cached
-  Aliases now display correctly as comma-separated list
-  Studio IDs properly converted from numeric to string

API Integration:
- Base URL: https://api.theporndb.net
- Authentication: Bearer token via TPDB_API_KEY env var
- Endpoints: /performers, /sites, /scenes
- Rate limiting handled with warnings

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-14 22:04:23 -05:00

542 lines
14 KiB
Go

package main
import (
"context"
"fmt"
"os"
"github.com/spf13/cobra"
"git.leaktechnologies.dev/stu/Goondex/internal/db"
"git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb"
)
var (
dbPath string
rootCmd = &cobra.Command{
Use: "goondex",
Short: "Goondex - Fast, local-first media indexer",
Long: `Goondex is a fast, local-first media indexer for adult content that ingests metadata from external sources.`,
}
)
func init() {
rootCmd.PersistentFlags().StringVar(&dbPath, "db", "./goondex.db", "Path to SQLite database")
// Add subcommands
rootCmd.AddCommand(performerSearchCmd)
rootCmd.AddCommand(studioSearchCmd)
rootCmd.AddCommand(sceneSearchCmd)
rootCmd.AddCommand(importCmd)
rootCmd.AddCommand(versionCmd)
}
// Import command with subcommands
var importCmd = &cobra.Command{
Use: "import",
Short: "Import data from external sources (TPDB)",
Long: `Import performers, studios, and scenes from ThePornDB into your local database.`,
}
func init() {
importCmd.AddCommand(importPerformerCmd)
importCmd.AddCommand(importStudioCmd)
importCmd.AddCommand(importSceneCmd)
}
func main() {
if err := rootCmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func getDB() (*db.DB, error) {
database, err := db.Open(dbPath)
if err != nil {
return nil, fmt.Errorf("failed to open database: %w", err)
}
return database, nil
}
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version information",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("Goondex v0.1.0-dev2")
},
}
var performerSearchCmd = &cobra.Command{
Use: "performer-search [query]",
Short: "Search for performers (auto-fetches from TPDB if not in local database)",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
store := db.NewPerformerStore(database)
performers, err := store.Search(query)
if err != nil {
return fmt.Errorf("search failed: %w", err)
}
// If no local results, try fetching from TPDB
if len(performers) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
return nil
}
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
tpdbPerformers, err := scraper.SearchPerformers(context.Background(), query)
if err != nil {
fmt.Printf("⚠ TPDB search failed: %v\n", err)
return nil
}
if len(tpdbPerformers) == 0 {
fmt.Println("No performers found on TPDB either.")
return nil
}
fmt.Printf("Found %d performer(s) on TPDB. Importing...\n\n", len(tpdbPerformers))
// Import from TPDB
imported := 0
for _, p := range tpdbPerformers {
if err := store.Create(&p); err != nil {
fmt.Printf("⚠ Failed to import %s: %v\n", p.Name, err)
continue
}
imported++
}
// Search again to get the imported performers with their IDs
performers, err = store.Search(query)
if err != nil {
return fmt.Errorf("search failed after import: %w", err)
}
fmt.Printf("✓ Imported %d performer(s)\n\n", imported)
}
fmt.Printf("Found %d performer(s):\n\n", len(performers))
for _, p := range performers {
fmt.Printf("ID: %d\n", p.ID)
fmt.Printf("Name: %s\n", p.Name)
if p.Aliases != "" {
fmt.Printf("Aliases: %s\n", p.Aliases)
}
if p.Country != "" {
fmt.Printf("Country: %s\n", p.Country)
}
if p.Gender != "" {
fmt.Printf("Gender: %s\n", p.Gender)
}
if p.Source != "" {
fmt.Printf("Source: %s (ID: %s)\n", p.Source, p.SourceID)
}
fmt.Println("---")
}
return nil
},
}
var studioSearchCmd = &cobra.Command{
Use: "studio-search [query]",
Short: "Search for studios (auto-fetches from TPDB if not in local database)",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
store := db.NewStudioStore(database)
studios, err := store.Search(query)
if err != nil {
return fmt.Errorf("search failed: %w", err)
}
// If no local results, try fetching from TPDB
if len(studios) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
return nil
}
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
tpdbStudios, err := scraper.SearchStudios(context.Background(), query)
if err != nil {
fmt.Printf("⚠ TPDB search failed: %v\n", err)
return nil
}
if len(tpdbStudios) == 0 {
fmt.Println("No studios found on TPDB either.")
return nil
}
fmt.Printf("Found %d studio(s) on TPDB. Importing...\n\n", len(tpdbStudios))
// Import from TPDB
imported := 0
for _, s := range tpdbStudios {
if err := store.Create(&s); err != nil {
fmt.Printf("⚠ Failed to import %s: %v\n", s.Name, err)
continue
}
imported++
}
// Search again to get the imported studios with their IDs
studios, err = store.Search(query)
if err != nil {
return fmt.Errorf("search failed after import: %w", err)
}
fmt.Printf("✓ Imported %d studio(s)\n\n", imported)
}
fmt.Printf("Found %d studio(s):\n\n", len(studios))
for _, s := range studios {
fmt.Printf("ID: %d\n", s.ID)
fmt.Printf("Name: %s\n", s.Name)
if s.Description != "" {
fmt.Printf("Description: %s\n", s.Description)
}
if s.Source != "" {
fmt.Printf("Source: %s (ID: %s)\n", s.Source, s.SourceID)
}
fmt.Println("---")
}
return nil
},
}
var sceneSearchCmd = &cobra.Command{
Use: "scene-search [query]",
Short: "Search for scenes (auto-fetches from TPDB if not in local database)",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
store := db.NewSceneStore(database)
scenes, err := store.Search(query)
if err != nil {
return fmt.Errorf("search failed: %w", err)
}
// If no local results, try fetching from TPDB
if len(scenes) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
return nil
}
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
tpdbScenes, err := scraper.SearchScenes(context.Background(), query)
if err != nil {
fmt.Printf("⚠ TPDB search failed: %v\n", err)
return nil
}
if len(tpdbScenes) == 0 {
fmt.Println("No scenes found on TPDB either.")
return nil
}
fmt.Printf("Found %d scene(s) on TPDB. Importing (basic metadata only)...\n\n", len(tpdbScenes))
// Import scenes (simplified - just scene metadata, no relationships)
imported := 0
for _, sc := range tpdbScenes {
// Clear relationships to avoid complexity in auto-import
sc.Performers = nil
sc.Tags = nil
sc.Studio = nil
sc.StudioID = nil
if err := store.Create(&sc); err != nil {
fmt.Printf("⚠ Failed to import %s: %v\n", sc.Title, err)
continue
}
imported++
}
// Search again to get the imported scenes with their IDs
scenes, err = store.Search(query)
if err != nil {
return fmt.Errorf("search failed after import: %w", err)
}
fmt.Printf("✓ Imported %d scene(s) (use 'import scene' for full metadata with relationships)\n\n", imported)
}
fmt.Printf("Found %d scene(s):\n\n", len(scenes))
for _, sc := range scenes {
fmt.Printf("ID: %d\n", sc.ID)
fmt.Printf("Title: %s\n", sc.Title)
if sc.Code != "" {
fmt.Printf("Code: %s\n", sc.Code)
}
if sc.Date != "" {
fmt.Printf("Date: %s\n", sc.Date)
}
if sc.Description != "" {
fmt.Printf("Description: %s\n", sc.Description)
}
if sc.Source != "" {
fmt.Printf("Source: %s (ID: %s)\n", sc.Source, sc.SourceID)
}
fmt.Println("---")
}
return nil
},
}
var importPerformerCmd = &cobra.Command{
Use: "performer [query]",
Short: "Search TPDB for performers and import them to local database",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
}
// Create TPDB scraper
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
// Search TPDB
fmt.Printf("Searching TPDB for performers matching '%s'...\n", query)
performers, err := scraper.SearchPerformers(context.Background(), query)
if err != nil {
return fmt.Errorf("failed to search TPDB: %w", err)
}
if len(performers) == 0 {
fmt.Println("No performers found on TPDB")
return nil
}
fmt.Printf("Found %d performer(s) on TPDB\n\n", len(performers))
// Open database
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
store := db.NewPerformerStore(database)
// Import each performer
imported := 0
for _, p := range performers {
fmt.Printf("Importing: %s (TPDB ID: %s)\n", p.Name, p.SourceID)
if err := store.Create(&p); err != nil {
fmt.Printf(" ⚠ Warning: Failed to import: %v\n", err)
continue
}
fmt.Printf(" ✓ Imported with local ID: %d\n", p.ID)
imported++
}
fmt.Printf("\n✓ Successfully imported %d/%d performers\n", imported, len(performers))
return nil
},
}
var importStudioCmd = &cobra.Command{
Use: "studio [query]",
Short: "Search TPDB for studios and import them to local database",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
}
// Create TPDB scraper
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
// Search TPDB
fmt.Printf("Searching TPDB for studios matching '%s'...\n", query)
studios, err := scraper.SearchStudios(context.Background(), query)
if err != nil {
return fmt.Errorf("failed to search TPDB: %w", err)
}
if len(studios) == 0 {
fmt.Println("No studios found on TPDB")
return nil
}
fmt.Printf("Found %d studio(s) on TPDB\n\n", len(studios))
// Open database
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
store := db.NewStudioStore(database)
// Import each studio
imported := 0
for _, s := range studios {
fmt.Printf("Importing: %s (TPDB ID: %s)\n", s.Name, s.SourceID)
if err := store.Create(&s); err != nil {
fmt.Printf(" ⚠ Warning: Failed to import: %v\n", err)
continue
}
fmt.Printf(" ✓ Imported with local ID: %d\n", s.ID)
imported++
}
fmt.Printf("\n✓ Successfully imported %d/%d studios\n", imported, len(studios))
return nil
},
}
var importSceneCmd = &cobra.Command{
Use: "scene [query]",
Short: "Search TPDB for scenes and import them to local database",
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
}
// Create TPDB scraper
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
// Search TPDB
fmt.Printf("Searching TPDB for scenes matching '%s'...\n", query)
scenes, err := scraper.SearchScenes(context.Background(), query)
if err != nil {
return fmt.Errorf("failed to search TPDB: %w", err)
}
if len(scenes) == 0 {
fmt.Println("No scenes found on TPDB")
return nil
}
fmt.Printf("Found %d scene(s) on TPDB\n\n", len(scenes))
// Open database
database, err := getDB()
if err != nil {
return err
}
defer database.Close()
sceneStore := db.NewSceneStore(database)
performerStore := db.NewPerformerStore(database)
studioStore := db.NewStudioStore(database)
tagStore := db.NewTagStore(database)
// Import each scene
imported := 0
for _, sc := range scenes {
fmt.Printf("Importing: %s (TPDB ID: %s)\n", sc.Title, sc.SourceID)
// Import studio if present
if sc.Studio != nil {
if err := studioStore.Create(sc.Studio); err != nil {
// Studio might already exist, try to fetch it
studios, _ := studioStore.Search(sc.Studio.Name)
if len(studios) > 0 {
sc.StudioID = &studios[0].ID
}
} else {
sc.StudioID = &sc.Studio.ID
}
}
// Create scene
if err := sceneStore.Create(&sc); err != nil {
fmt.Printf(" ⚠ Warning: Failed to import scene: %v\n", err)
continue
}
// Import and link performers
for _, p := range sc.Performers {
if err := performerStore.Create(&p); err != nil {
// Performer might already exist
performers, _ := performerStore.Search(p.Name)
if len(performers) > 0 {
p.ID = performers[0].ID
}
}
if p.ID > 0 {
sceneStore.AddPerformer(sc.ID, p.ID)
}
}
// Import and link tags
for _, t := range sc.Tags {
existing, _ := tagStore.GetByName(t.Name)
if existing != nil {
t.ID = existing.ID
} else {
if err := tagStore.Create(&t); err != nil {
continue
}
}
if t.ID > 0 {
sceneStore.AddTag(sc.ID, t.ID)
}
}
fmt.Printf(" ✓ Imported with local ID: %d\n", sc.ID)
imported++
}
fmt.Printf("\n✓ Successfully imported %d/%d scenes\n", imported, len(scenes))
return nil
},
}