From 324095c9e91b9020aec65d20957c877d66a2d511 Mon Sep 17 00:00:00 2001 From: Stu Leak Date: Mon, 17 Nov 2025 14:38:58 -0500 Subject: [PATCH] Prepare v0.1.0-dev5 --- CHANGELOG.md | 12 +++ README.md | 18 ++++ cmd/goondex/main.go | 177 +++++++++++++++++++++++++---------- docs/ADULT_EMPIRE_SCRAPER.md | 9 +- 4 files changed, 163 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a74755..a58b650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## v0.1.0-dev5 (2025-11-17) + +### 🧰 CLI & Import Improvements +- Added `goondex import all` plus explicit `all-performers`, `all-studios`, and `all-scenes` bulk commands so the entire TPDB catalog can be ingested with one invocation. +- Each bulk import now exposes start/max page flags even when running the combined command, making resume workflows painless. +- Standardized TPDB API key handling across commands; errors now include clear export instructions via the shared helper. + +### πŸ“– Developer Experience +- `README.md` now documents when/how to rebuild the CLI binary, preventing stale builds after source changes. +- `docs/ADULT_EMPIRE_SCRAPER.md` metadata bumped to v0.1.0-dev5. +- `goondex version` output updated to reflect the new release and highlight the bulk import capability. + ## v0.1.0-dev4 (2025-11-16) ### 🎨 Web UI Enhancements diff --git a/README.md b/README.md index f101686..3ed14f8 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,24 @@ go build -o bin/goondex ./cmd/goondex go run ./cmd/goondex performer-search "test" ``` +### Building & Rebuilding the CLI + +The Goondex binary is not rebuilt automaticallyβ€”whenever you change Go files (especially under `cmd/goondex` or `internal/*`), rebuild before re-running commands. + +```bash +# Clean out any previous binary (prevents running stale code) +rm -f goondex bin/goondex + +# Build the latest CLI +go build -o goondex ./cmd/goondex +``` + +After rebuilding, rerun `./goondex` (or the binary under `bin/`) so new commands like `import all` become available. Repeat the build whenever: +- You pull new commits (e.g., moving to v0.1.0-dev5) +- CLI command definitions change +- Shared packages under `internal/` are modified +- You switch Go versions or modules are updated (`go mod tidy`, `go get`, etc.) + ## Contributing This is a personal project, but contributions are welcome! Please open an issue before submitting large changes. diff --git a/cmd/goondex/main.go b/cmd/goondex/main.go index 5aa3191..6879b9d 100644 --- a/cmd/goondex/main.go +++ b/cmd/goondex/main.go @@ -4,10 +4,10 @@ import ( "context" "fmt" "os" + "strconv" "time" - "github.com/spf13/cobra" "git.leaktechnologies.dev/stu/Goondex/internal/db" "git.leaktechnologies.dev/stu/Goondex/internal/model" "git.leaktechnologies.dev/stu/Goondex/internal/scraper/adultemp" @@ -15,8 +15,11 @@ import ( "git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb" "git.leaktechnologies.dev/stu/Goondex/internal/sync" "git.leaktechnologies.dev/stu/Goondex/internal/web" + "github.com/spf13/cobra" ) +const tpdbAPIKeyEnvVar = "TPDB_API_KEY" + var ( dbPath string rootCmd = &cobra.Command{ @@ -59,6 +62,64 @@ var importCmd = &cobra.Command{ Long: `Import performers, studios, and scenes from ThePornDB into your local database.`, } +var importAllCmd = &cobra.Command{ + Use: "all", + Short: "Import all TPDB performers, studios, and scenes", + Long: `Run the performer, studio, and scene bulk imports sequentially to ingest the entire TPDB catalog. + +Use the per-entity pagination flags to resume or limit parts of the import.`, + RunE: func(cmd *cobra.Command, args []string) error { + performerStart, _ := cmd.Flags().GetInt("performer-start-page") + performerMax, _ := cmd.Flags().GetInt("performer-max-pages") + studioStart, _ := cmd.Flags().GetInt("studio-start-page") + studioMax, _ := cmd.Flags().GetInt("studio-max-pages") + sceneStart, _ := cmd.Flags().GetInt("scene-start-page") + sceneMax, _ := cmd.Flags().GetInt("scene-max-pages") + + // Propagate flag values to the existing per-entity commands + if err := importAllPerformersCmd.Flags().Set("start-page", strconv.Itoa(performerStart)); err != nil { + return fmt.Errorf("failed to set performer start page: %w", err) + } + if err := importAllPerformersCmd.Flags().Set("max-pages", strconv.Itoa(performerMax)); err != nil { + return fmt.Errorf("failed to set performer max pages: %w", err) + } + if err := importAllStudiosCmd.Flags().Set("start-page", strconv.Itoa(studioStart)); err != nil { + return fmt.Errorf("failed to set studio start page: %w", err) + } + if err := importAllStudiosCmd.Flags().Set("max-pages", strconv.Itoa(studioMax)); err != nil { + return fmt.Errorf("failed to set studio max pages: %w", err) + } + if err := importAllScenesCmd.Flags().Set("start-page", strconv.Itoa(sceneStart)); err != nil { + return fmt.Errorf("failed to set scene start page: %w", err) + } + if err := importAllScenesCmd.Flags().Set("max-pages", strconv.Itoa(sceneMax)); err != nil { + return fmt.Errorf("failed to set scene max pages: %w", err) + } + + fmt.Println("╔═══════════════════════════════════════════════════════════════╗") + fmt.Println("β•‘ TPDB BULK IMPORT - ALL ENTITIES β•‘") + fmt.Println("β•šβ•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•β•") + + fmt.Println("\n➑ Importing performers...") + if err := importAllPerformersCmd.RunE(importAllPerformersCmd, []string{}); err != nil { + return fmt.Errorf("performer import failed: %w", err) + } + + fmt.Println("\n➑ Importing studios...") + if err := importAllStudiosCmd.RunE(importAllStudiosCmd, []string{}); err != nil { + return fmt.Errorf("studio import failed: %w", err) + } + + fmt.Println("\n➑ Importing scenes...") + if err := importAllScenesCmd.RunE(importAllScenesCmd, []string{}); err != nil { + return fmt.Errorf("scene import failed: %w", err) + } + + fmt.Println("\nβœ“ Completed TPDB bulk import for performers, studios, and scenes") + return nil + }, +} + func init() { importCmd.AddCommand(importPerformerCmd) importCmd.AddCommand(importAllPerformersCmd) @@ -67,6 +128,7 @@ func init() { importCmd.AddCommand(importSceneCmd) importCmd.AddCommand(importAllScenesCmd) importCmd.AddCommand(importMovieCmd) + importCmd.AddCommand(importAllCmd) // Flags for bulk import importAllPerformersCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)") @@ -75,6 +137,12 @@ func init() { importAllStudiosCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)") importAllScenesCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)") importAllScenesCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)") + importAllCmd.Flags().Int("performer-start-page", 1, "Performer import start page (for resuming)") + importAllCmd.Flags().Int("performer-max-pages", 0, "Maximum performer pages to import (0 = all)") + importAllCmd.Flags().Int("studio-start-page", 1, "Studio import start page (for resuming)") + importAllCmd.Flags().Int("studio-max-pages", 0, "Maximum studio pages to import (0 = all)") + importAllCmd.Flags().Int("scene-start-page", 1, "Scene import start page (for resuming)") + importAllCmd.Flags().Int("scene-max-pages", 0, "Maximum scene pages to import (0 = all)") // Movie import flags importMovieCmd.Flags().String("source", "adultemp", "Source to import from (adultemp)") @@ -114,9 +182,9 @@ var syncAllCmd = &cobra.Command{ force, _ := cmd.Flags().GetBool("force") interval, _ := cmd.Flags().GetDuration("interval") - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } database, err := getDB() @@ -173,9 +241,9 @@ var syncPerformersCmd = &cobra.Command{ force, _ := cmd.Flags().GetBool("force") interval, _ := cmd.Flags().GetDuration("interval") - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } database, err := getDB() @@ -222,9 +290,9 @@ var syncStudiosCmd = &cobra.Command{ force, _ := cmd.Flags().GetBool("force") interval, _ := cmd.Flags().GetDuration("interval") - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } database, err := getDB() @@ -271,9 +339,9 @@ var syncScenesCmd = &cobra.Command{ force, _ := cmd.Flags().GetBool("force") interval, _ := cmd.Flags().GetDuration("interval") - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } database, err := getDB() @@ -376,6 +444,18 @@ func formatDuration(d time.Duration) string { return fmt.Sprintf("%d days", int(d.Hours()/24)) } +func getTPDBAPIKey() (string, error) { + apiKey := os.Getenv(tpdbAPIKeyEnvVar) + if apiKey == "" { + return "", fmt.Errorf("%s environment variable is not set.\n%s", tpdbAPIKeyEnvVar, apiKeySetupInstructions()) + } + return apiKey, nil +} + +func apiKeySetupInstructions() string { + return fmt.Sprintf("Set it by running:\n\n export %s=\"your-api-key\"", tpdbAPIKeyEnvVar) +} + func main() { if err := rootCmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -417,9 +497,9 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print version information", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("Goondex v0.1.0-dev4") + fmt.Println("Goondex v0.1.0-dev5") fmt.Println("Features:") - fmt.Println(" β€’ TPDB integration with auto-import") + fmt.Println(" β€’ TPDB integration with auto-import & bulk commands") fmt.Println(" β€’ Adult Empire scraper (scenes & performers)") fmt.Println(" β€’ Multi-source data merging") fmt.Println(" β€’ Grid-based web UI with GX components") @@ -451,10 +531,9 @@ var performerSearchCmd = &cobra.Command{ if len(performers) == 0 { fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query) - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.") - fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"") + apiKey, err := getTPDBAPIKey() + if err != nil { + fmt.Printf("⚠ %s\n", err) return nil } @@ -638,8 +717,10 @@ var performerUpdateCmd = &cobra.Command{ // Update from TPDB if available if performer.Source == "tpdb" && performer.SourceID != "" { - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey != "" { + apiKey, err := getTPDBAPIKey() + if err != nil { + fmt.Printf("⚠ %s\n", err) + } else { fmt.Println("πŸ“₯ Fetching latest data from TPDB...") scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey) @@ -657,8 +738,6 @@ var performerUpdateCmd = &cobra.Command{ performer = updatedPerformer } } - } else { - fmt.Println("⚠ TPDB_API_KEY not set, skipping TPDB update") } } @@ -858,10 +937,9 @@ var studioSearchCmd = &cobra.Command{ if len(studios) == 0 { fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query) - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.") - fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"") + apiKey, err := getTPDBAPIKey() + if err != nil { + fmt.Printf("⚠ %s\n", err) return nil } @@ -1022,10 +1100,9 @@ var sceneSearchCmd = &cobra.Command{ if len(scenes) == 0 { fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query) - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.") - fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"") + apiKey, err := getTPDBAPIKey() + if err != nil { + fmt.Printf("⚠ %s\n", err) return nil } @@ -1224,9 +1301,9 @@ var importAllPerformersCmd = &cobra.Command{ maxPages, _ := cmd.Flags().GetInt("max-pages") // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -1336,9 +1413,9 @@ var importAllStudiosCmd = &cobra.Command{ maxPages, _ := cmd.Flags().GetInt("max-pages") // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -1454,9 +1531,9 @@ var importAllScenesCmd = &cobra.Command{ maxPages, _ := cmd.Flags().GetInt("max-pages") // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -1612,9 +1689,9 @@ var importPerformerCmd = &cobra.Command{ query := args[0] // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -1668,9 +1745,9 @@ var importStudioCmd = &cobra.Command{ query := args[0] // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -2166,9 +2243,9 @@ var importSceneCmd = &cobra.Command{ query := args[0] // Get API key from environment - apiKey := os.Getenv("TPDB_API_KEY") - if apiKey == "" { - return fmt.Errorf("TPDB_API_KEY environment variable is not set") + apiKey, err := getTPDBAPIKey() + if err != nil { + return err } // Create TPDB scraper @@ -2272,7 +2349,7 @@ Movies can be imported by: 2. Direct URL: ./goondex import movie https://www.adultdvdempire.com/... Note: For bulk movie import, movies are best imported through Adult Empire's catalog.`, - Args: cobra.MinimumNArgs(1), + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { input := args[0] diff --git a/docs/ADULT_EMPIRE_SCRAPER.md b/docs/ADULT_EMPIRE_SCRAPER.md index 61da1a1..63588fd 100644 --- a/docs/ADULT_EMPIRE_SCRAPER.md +++ b/docs/ADULT_EMPIRE_SCRAPER.md @@ -1,7 +1,7 @@ # Adult Empire Scraper Integration -**Version**: v0.1.0-dev4 -**Last Updated**: 2025-11-16 +**Version**: v0.1.0-dev5 +**Last Updated**: 2025-11-17 ## Overview @@ -316,6 +316,9 @@ To improve Adult Empire scraping: ## Version History +- **v0.1.0-dev5** (2025-11-17): Documentation refresh for TPDB bulk-import release + - Updated version metadata and changelog references + - Clarified rebuild steps for the CLI additions - **v0.1.0-dev4** (2025-11-16): Initial Adult Empire scraper implementation - HTTP client with cookie support - XPath parsing utilities @@ -325,5 +328,5 @@ To improve Adult Empire scraping: --- -**Last Updated**: 2025-11-16 +**Last Updated**: 2025-11-17 **Maintainer**: Goondex Team