Major Features: - ✅ Complete TPDB scraper implementation with real API calls - ✅ Auto-fetch on cache miss: search commands now automatically import from TPDB when not found locally - ✅ Comprehensive documentation (5 markdown files for Bookstack) - ✅ Import commands for performers, studios, and scenes - ✅ Fixed JSON type mismatches (aliases array, studio numeric IDs) Changes: 1. TPDB Scraper (internal/scraper/tpdb/): - types.go: Full API response structures with correct types - PerformerResponse.Aliases: string → []string (TPDB returns array) - StudioResponse.ID: string → int (TPDB returns numeric IDs) - SiteInfo.ID: string → int (scenes reference studios by number) - mapper.go: Maps TPDB responses to internal models - Converts aliases array to comma-separated string - Converts numeric studio IDs to strings using strconv.Itoa() - scraper.go: Real HTTP client with Bearer token auth - SearchPerformers, SearchStudios, SearchScenes implemented - GetPerformerByID, GetStudioByID, GetSceneByID implemented 2. CLI Auto-Fetch (cmd/goondex/main.go): - performer-search: Auto-fetches from TPDB if local DB empty - studio-search: Auto-fetches from TPDB if local DB empty - scene-search: Auto-fetches basic metadata (no relationships) - Graceful handling of missing TPDB_API_KEY - Import → search again to get local IDs 3. Documentation (docs/): - INDEX.md: Documentation overview and navigation - ARCHITECTURE.md: System design, data flow, component diagrams - DATABASE_SCHEMA.md: Complete schema with relationships and indexes - CLI_REFERENCE.md: All commands with examples - TPDB_INTEGRATION.md: API guide, data mapping, best practices 4. Fixes: - .gitignore: Fixed pattern to allow cmd/goondex/* and cmd/goondexd/* - README: Updated to reflect TPDB integration and auto-fetch Testing: - ✅ performer-search "Riley Reid" - auto-fetched 2 performers, cached - ✅ studio-search "Brazzers" - auto-fetched 12 studios, cached - ✅ Aliases now display correctly as comma-separated list - ✅ Studio IDs properly converted from numeric to string API Integration: - Base URL: https://api.theporndb.net - Authentication: Bearer token via TPDB_API_KEY env var - Endpoints: /performers, /sites, /scenes - Rate limiting handled with warnings 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
542 lines
14 KiB
Go
542 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/spf13/cobra"
|
|
"git.leaktechnologies.dev/stu/Goondex/internal/db"
|
|
"git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb"
|
|
)
|
|
|
|
var (
|
|
dbPath string
|
|
rootCmd = &cobra.Command{
|
|
Use: "goondex",
|
|
Short: "Goondex - Fast, local-first media indexer",
|
|
Long: `Goondex is a fast, local-first media indexer for adult content that ingests metadata from external sources.`,
|
|
}
|
|
)
|
|
|
|
func init() {
|
|
rootCmd.PersistentFlags().StringVar(&dbPath, "db", "./goondex.db", "Path to SQLite database")
|
|
|
|
// Add subcommands
|
|
rootCmd.AddCommand(performerSearchCmd)
|
|
rootCmd.AddCommand(studioSearchCmd)
|
|
rootCmd.AddCommand(sceneSearchCmd)
|
|
rootCmd.AddCommand(importCmd)
|
|
rootCmd.AddCommand(versionCmd)
|
|
}
|
|
|
|
// Import command with subcommands
|
|
var importCmd = &cobra.Command{
|
|
Use: "import",
|
|
Short: "Import data from external sources (TPDB)",
|
|
Long: `Import performers, studios, and scenes from ThePornDB into your local database.`,
|
|
}
|
|
|
|
func init() {
|
|
importCmd.AddCommand(importPerformerCmd)
|
|
importCmd.AddCommand(importStudioCmd)
|
|
importCmd.AddCommand(importSceneCmd)
|
|
}
|
|
|
|
func main() {
|
|
if err := rootCmd.Execute(); err != nil {
|
|
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func getDB() (*db.DB, error) {
|
|
database, err := db.Open(dbPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open database: %w", err)
|
|
}
|
|
return database, nil
|
|
}
|
|
|
|
var versionCmd = &cobra.Command{
|
|
Use: "version",
|
|
Short: "Print version information",
|
|
Run: func(cmd *cobra.Command, args []string) {
|
|
fmt.Println("Goondex v0.1.0-dev2")
|
|
},
|
|
}
|
|
|
|
var performerSearchCmd = &cobra.Command{
|
|
Use: "performer-search [query]",
|
|
Short: "Search for performers (auto-fetches from TPDB if not in local database)",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
store := db.NewPerformerStore(database)
|
|
performers, err := store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed: %w", err)
|
|
}
|
|
|
|
// If no local results, try fetching from TPDB
|
|
if len(performers) == 0 {
|
|
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
|
|
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
|
|
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
|
|
return nil
|
|
}
|
|
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
tpdbPerformers, err := scraper.SearchPerformers(context.Background(), query)
|
|
if err != nil {
|
|
fmt.Printf("⚠ TPDB search failed: %v\n", err)
|
|
return nil
|
|
}
|
|
|
|
if len(tpdbPerformers) == 0 {
|
|
fmt.Println("No performers found on TPDB either.")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d performer(s) on TPDB. Importing...\n\n", len(tpdbPerformers))
|
|
|
|
// Import from TPDB
|
|
imported := 0
|
|
for _, p := range tpdbPerformers {
|
|
if err := store.Create(&p); err != nil {
|
|
fmt.Printf("⚠ Failed to import %s: %v\n", p.Name, err)
|
|
continue
|
|
}
|
|
imported++
|
|
}
|
|
|
|
// Search again to get the imported performers with their IDs
|
|
performers, err = store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed after import: %w", err)
|
|
}
|
|
|
|
fmt.Printf("✓ Imported %d performer(s)\n\n", imported)
|
|
}
|
|
|
|
fmt.Printf("Found %d performer(s):\n\n", len(performers))
|
|
for _, p := range performers {
|
|
fmt.Printf("ID: %d\n", p.ID)
|
|
fmt.Printf("Name: %s\n", p.Name)
|
|
if p.Aliases != "" {
|
|
fmt.Printf("Aliases: %s\n", p.Aliases)
|
|
}
|
|
if p.Country != "" {
|
|
fmt.Printf("Country: %s\n", p.Country)
|
|
}
|
|
if p.Gender != "" {
|
|
fmt.Printf("Gender: %s\n", p.Gender)
|
|
}
|
|
if p.Source != "" {
|
|
fmt.Printf("Source: %s (ID: %s)\n", p.Source, p.SourceID)
|
|
}
|
|
fmt.Println("---")
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var studioSearchCmd = &cobra.Command{
|
|
Use: "studio-search [query]",
|
|
Short: "Search for studios (auto-fetches from TPDB if not in local database)",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
store := db.NewStudioStore(database)
|
|
studios, err := store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed: %w", err)
|
|
}
|
|
|
|
// If no local results, try fetching from TPDB
|
|
if len(studios) == 0 {
|
|
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
|
|
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
|
|
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
|
|
return nil
|
|
}
|
|
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
tpdbStudios, err := scraper.SearchStudios(context.Background(), query)
|
|
if err != nil {
|
|
fmt.Printf("⚠ TPDB search failed: %v\n", err)
|
|
return nil
|
|
}
|
|
|
|
if len(tpdbStudios) == 0 {
|
|
fmt.Println("No studios found on TPDB either.")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d studio(s) on TPDB. Importing...\n\n", len(tpdbStudios))
|
|
|
|
// Import from TPDB
|
|
imported := 0
|
|
for _, s := range tpdbStudios {
|
|
if err := store.Create(&s); err != nil {
|
|
fmt.Printf("⚠ Failed to import %s: %v\n", s.Name, err)
|
|
continue
|
|
}
|
|
imported++
|
|
}
|
|
|
|
// Search again to get the imported studios with their IDs
|
|
studios, err = store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed after import: %w", err)
|
|
}
|
|
|
|
fmt.Printf("✓ Imported %d studio(s)\n\n", imported)
|
|
}
|
|
|
|
fmt.Printf("Found %d studio(s):\n\n", len(studios))
|
|
for _, s := range studios {
|
|
fmt.Printf("ID: %d\n", s.ID)
|
|
fmt.Printf("Name: %s\n", s.Name)
|
|
if s.Description != "" {
|
|
fmt.Printf("Description: %s\n", s.Description)
|
|
}
|
|
if s.Source != "" {
|
|
fmt.Printf("Source: %s (ID: %s)\n", s.Source, s.SourceID)
|
|
}
|
|
fmt.Println("---")
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var sceneSearchCmd = &cobra.Command{
|
|
Use: "scene-search [query]",
|
|
Short: "Search for scenes (auto-fetches from TPDB if not in local database)",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
store := db.NewSceneStore(database)
|
|
scenes, err := store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed: %w", err)
|
|
}
|
|
|
|
// If no local results, try fetching from TPDB
|
|
if len(scenes) == 0 {
|
|
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
|
|
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
|
|
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
|
|
return nil
|
|
}
|
|
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
tpdbScenes, err := scraper.SearchScenes(context.Background(), query)
|
|
if err != nil {
|
|
fmt.Printf("⚠ TPDB search failed: %v\n", err)
|
|
return nil
|
|
}
|
|
|
|
if len(tpdbScenes) == 0 {
|
|
fmt.Println("No scenes found on TPDB either.")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d scene(s) on TPDB. Importing (basic metadata only)...\n\n", len(tpdbScenes))
|
|
|
|
// Import scenes (simplified - just scene metadata, no relationships)
|
|
imported := 0
|
|
for _, sc := range tpdbScenes {
|
|
// Clear relationships to avoid complexity in auto-import
|
|
sc.Performers = nil
|
|
sc.Tags = nil
|
|
sc.Studio = nil
|
|
sc.StudioID = nil
|
|
|
|
if err := store.Create(&sc); err != nil {
|
|
fmt.Printf("⚠ Failed to import %s: %v\n", sc.Title, err)
|
|
continue
|
|
}
|
|
imported++
|
|
}
|
|
|
|
// Search again to get the imported scenes with their IDs
|
|
scenes, err = store.Search(query)
|
|
if err != nil {
|
|
return fmt.Errorf("search failed after import: %w", err)
|
|
}
|
|
|
|
fmt.Printf("✓ Imported %d scene(s) (use 'import scene' for full metadata with relationships)\n\n", imported)
|
|
}
|
|
|
|
fmt.Printf("Found %d scene(s):\n\n", len(scenes))
|
|
for _, sc := range scenes {
|
|
fmt.Printf("ID: %d\n", sc.ID)
|
|
fmt.Printf("Title: %s\n", sc.Title)
|
|
if sc.Code != "" {
|
|
fmt.Printf("Code: %s\n", sc.Code)
|
|
}
|
|
if sc.Date != "" {
|
|
fmt.Printf("Date: %s\n", sc.Date)
|
|
}
|
|
if sc.Description != "" {
|
|
fmt.Printf("Description: %s\n", sc.Description)
|
|
}
|
|
if sc.Source != "" {
|
|
fmt.Printf("Source: %s (ID: %s)\n", sc.Source, sc.SourceID)
|
|
}
|
|
fmt.Println("---")
|
|
}
|
|
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var importPerformerCmd = &cobra.Command{
|
|
Use: "performer [query]",
|
|
Short: "Search TPDB for performers and import them to local database",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
// Get API key from environment
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
|
|
}
|
|
|
|
// Create TPDB scraper
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
|
|
// Search TPDB
|
|
fmt.Printf("Searching TPDB for performers matching '%s'...\n", query)
|
|
performers, err := scraper.SearchPerformers(context.Background(), query)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to search TPDB: %w", err)
|
|
}
|
|
|
|
if len(performers) == 0 {
|
|
fmt.Println("No performers found on TPDB")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d performer(s) on TPDB\n\n", len(performers))
|
|
|
|
// Open database
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
store := db.NewPerformerStore(database)
|
|
|
|
// Import each performer
|
|
imported := 0
|
|
for _, p := range performers {
|
|
fmt.Printf("Importing: %s (TPDB ID: %s)\n", p.Name, p.SourceID)
|
|
if err := store.Create(&p); err != nil {
|
|
fmt.Printf(" ⚠ Warning: Failed to import: %v\n", err)
|
|
continue
|
|
}
|
|
fmt.Printf(" ✓ Imported with local ID: %d\n", p.ID)
|
|
imported++
|
|
}
|
|
|
|
fmt.Printf("\n✓ Successfully imported %d/%d performers\n", imported, len(performers))
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var importStudioCmd = &cobra.Command{
|
|
Use: "studio [query]",
|
|
Short: "Search TPDB for studios and import them to local database",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
// Get API key from environment
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
|
|
}
|
|
|
|
// Create TPDB scraper
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
|
|
// Search TPDB
|
|
fmt.Printf("Searching TPDB for studios matching '%s'...\n", query)
|
|
studios, err := scraper.SearchStudios(context.Background(), query)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to search TPDB: %w", err)
|
|
}
|
|
|
|
if len(studios) == 0 {
|
|
fmt.Println("No studios found on TPDB")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d studio(s) on TPDB\n\n", len(studios))
|
|
|
|
// Open database
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
store := db.NewStudioStore(database)
|
|
|
|
// Import each studio
|
|
imported := 0
|
|
for _, s := range studios {
|
|
fmt.Printf("Importing: %s (TPDB ID: %s)\n", s.Name, s.SourceID)
|
|
if err := store.Create(&s); err != nil {
|
|
fmt.Printf(" ⚠ Warning: Failed to import: %v\n", err)
|
|
continue
|
|
}
|
|
fmt.Printf(" ✓ Imported with local ID: %d\n", s.ID)
|
|
imported++
|
|
}
|
|
|
|
fmt.Printf("\n✓ Successfully imported %d/%d studios\n", imported, len(studios))
|
|
return nil
|
|
},
|
|
}
|
|
|
|
var importSceneCmd = &cobra.Command{
|
|
Use: "scene [query]",
|
|
Short: "Search TPDB for scenes and import them to local database",
|
|
Args: cobra.MinimumNArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
query := args[0]
|
|
|
|
// Get API key from environment
|
|
apiKey := os.Getenv("TPDB_API_KEY")
|
|
if apiKey == "" {
|
|
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
|
|
}
|
|
|
|
// Create TPDB scraper
|
|
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
|
|
|
|
// Search TPDB
|
|
fmt.Printf("Searching TPDB for scenes matching '%s'...\n", query)
|
|
scenes, err := scraper.SearchScenes(context.Background(), query)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to search TPDB: %w", err)
|
|
}
|
|
|
|
if len(scenes) == 0 {
|
|
fmt.Println("No scenes found on TPDB")
|
|
return nil
|
|
}
|
|
|
|
fmt.Printf("Found %d scene(s) on TPDB\n\n", len(scenes))
|
|
|
|
// Open database
|
|
database, err := getDB()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer database.Close()
|
|
|
|
sceneStore := db.NewSceneStore(database)
|
|
performerStore := db.NewPerformerStore(database)
|
|
studioStore := db.NewStudioStore(database)
|
|
tagStore := db.NewTagStore(database)
|
|
|
|
// Import each scene
|
|
imported := 0
|
|
for _, sc := range scenes {
|
|
fmt.Printf("Importing: %s (TPDB ID: %s)\n", sc.Title, sc.SourceID)
|
|
|
|
// Import studio if present
|
|
if sc.Studio != nil {
|
|
if err := studioStore.Create(sc.Studio); err != nil {
|
|
// Studio might already exist, try to fetch it
|
|
studios, _ := studioStore.Search(sc.Studio.Name)
|
|
if len(studios) > 0 {
|
|
sc.StudioID = &studios[0].ID
|
|
}
|
|
} else {
|
|
sc.StudioID = &sc.Studio.ID
|
|
}
|
|
}
|
|
|
|
// Create scene
|
|
if err := sceneStore.Create(&sc); err != nil {
|
|
fmt.Printf(" ⚠ Warning: Failed to import scene: %v\n", err)
|
|
continue
|
|
}
|
|
|
|
// Import and link performers
|
|
for _, p := range sc.Performers {
|
|
if err := performerStore.Create(&p); err != nil {
|
|
// Performer might already exist
|
|
performers, _ := performerStore.Search(p.Name)
|
|
if len(performers) > 0 {
|
|
p.ID = performers[0].ID
|
|
}
|
|
}
|
|
if p.ID > 0 {
|
|
sceneStore.AddPerformer(sc.ID, p.ID)
|
|
}
|
|
}
|
|
|
|
// Import and link tags
|
|
for _, t := range sc.Tags {
|
|
existing, _ := tagStore.GetByName(t.Name)
|
|
if existing != nil {
|
|
t.ID = existing.ID
|
|
} else {
|
|
if err := tagStore.Create(&t); err != nil {
|
|
continue
|
|
}
|
|
}
|
|
if t.ID > 0 {
|
|
sceneStore.AddTag(sc.ID, t.ID)
|
|
}
|
|
}
|
|
|
|
fmt.Printf(" ✓ Imported with local ID: %d\n", sc.ID)
|
|
imported++
|
|
}
|
|
|
|
fmt.Printf("\n✓ Successfully imported %d/%d scenes\n", imported, len(scenes))
|
|
return nil
|
|
},
|
|
}
|