Prepare v0.1.0-dev5

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v0.1.0-dev5 (2025-11-17)
+
+### 🧰 CLI & Import Improvements
+- Added `goondex import all` plus explicit `all-performers`, `all-studios`, and `all-scenes` bulk commands so the entire TPDB catalog can be ingested with one invocation.
+- Each bulk import now exposes start/max page flags even when running the combined command, making resume workflows painless.
+- Standardized TPDB API key handling across commands; errors now include clear export instructions via the shared helper.
+
+### 📖 Developer Experience
+- `README.md` now documents when/how to rebuild the CLI binary, preventing stale builds after source changes.
+- `docs/ADULT_EMPIRE_SCRAPER.md` metadata bumped to v0.1.0-dev5.
+- `goondex version` output updated to reflect the new release and highlight the bulk import capability.
+
 ## v0.1.0-dev4 (2025-11-16)
 
 ### 🎨 Web UI Enhancements

README.md

@@ -160,6 +160,24 @@ go build -o bin/goondex ./cmd/goondex
 go run ./cmd/goondex performer-search "test"
 ```
 
+### Building & Rebuilding the CLI
+
+The Goondex binary is not rebuilt automatically—whenever you change Go files (especially under `cmd/goondex` or `internal/*`), rebuild before re-running commands.
+
+```bash
+# Clean out any previous binary (prevents running stale code)
+rm -f goondex bin/goondex
+
+# Build the latest CLI
+go build -o goondex ./cmd/goondex
+```
+
+After rebuilding, rerun `./goondex` (or the binary under `bin/`) so new commands like `import all` become available. Repeat the build whenever:
+- You pull new commits (e.g., moving to v0.1.0-dev5)
+- CLI command definitions change
+- Shared packages under `internal/` are modified
+- You switch Go versions or modules are updated (`go mod tidy`, `go get`, etc.)
+
 ## Contributing
 
 This is a personal project, but contributions are welcome! Please open an issue before submitting large changes.

cmd/goondex/main.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strconv"
 
 	"time"
 
-	"github.com/spf13/cobra"
 	"git.leaktechnologies.dev/stu/Goondex/internal/db"
 	"git.leaktechnologies.dev/stu/Goondex/internal/model"
 	"git.leaktechnologies.dev/stu/Goondex/internal/scraper/adultemp"
@@ -15,8 +15,11 @@ import (
 	"git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb"
 	"git.leaktechnologies.dev/stu/Goondex/internal/sync"
 	"git.leaktechnologies.dev/stu/Goondex/internal/web"
+	"github.com/spf13/cobra"
 )
 
+const tpdbAPIKeyEnvVar = "TPDB_API_KEY"
+
 var (
 	dbPath  string
 	rootCmd = &cobra.Command{
@@ -59,6 +62,64 @@ var importCmd = &cobra.Command{
 	Long:  `Import performers, studios, and scenes from ThePornDB into your local database.`,
 }
 
+var importAllCmd = &cobra.Command{
+	Use:   "all",
+	Short: "Import all TPDB performers, studios, and scenes",
+	Long: `Run the performer, studio, and scene bulk imports sequentially to ingest the entire TPDB catalog.
+
+Use the per-entity pagination flags to resume or limit parts of the import.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		performerStart, _ := cmd.Flags().GetInt("performer-start-page")
+		performerMax, _ := cmd.Flags().GetInt("performer-max-pages")
+		studioStart, _ := cmd.Flags().GetInt("studio-start-page")
+		studioMax, _ := cmd.Flags().GetInt("studio-max-pages")
+		sceneStart, _ := cmd.Flags().GetInt("scene-start-page")
+		sceneMax, _ := cmd.Flags().GetInt("scene-max-pages")
+
+		// Propagate flag values to the existing per-entity commands
+		if err := importAllPerformersCmd.Flags().Set("start-page", strconv.Itoa(performerStart)); err != nil {
+			return fmt.Errorf("failed to set performer start page: %w", err)
+		}
+		if err := importAllPerformersCmd.Flags().Set("max-pages", strconv.Itoa(performerMax)); err != nil {
+			return fmt.Errorf("failed to set performer max pages: %w", err)
+		}
+		if err := importAllStudiosCmd.Flags().Set("start-page", strconv.Itoa(studioStart)); err != nil {
+			return fmt.Errorf("failed to set studio start page: %w", err)
+		}
+		if err := importAllStudiosCmd.Flags().Set("max-pages", strconv.Itoa(studioMax)); err != nil {
+			return fmt.Errorf("failed to set studio max pages: %w", err)
+		}
+		if err := importAllScenesCmd.Flags().Set("start-page", strconv.Itoa(sceneStart)); err != nil {
+			return fmt.Errorf("failed to set scene start page: %w", err)
+		}
+		if err := importAllScenesCmd.Flags().Set("max-pages", strconv.Itoa(sceneMax)); err != nil {
+			return fmt.Errorf("failed to set scene max pages: %w", err)
+		}
+
+		fmt.Println("╔═══════════════════════════════════════════════════════════════╗")
+		fmt.Println("║              TPDB BULK IMPORT - ALL ENTITIES                  ║")
+		fmt.Println("╚═══════════════════════════════════════════════════════════════╝")
+
+		fmt.Println("\n➡ Importing performers...")
+		if err := importAllPerformersCmd.RunE(importAllPerformersCmd, []string{}); err != nil {
+			return fmt.Errorf("performer import failed: %w", err)
+		}
+
+		fmt.Println("\n➡ Importing studios...")
+		if err := importAllStudiosCmd.RunE(importAllStudiosCmd, []string{}); err != nil {
+			return fmt.Errorf("studio import failed: %w", err)
+		}
+
+		fmt.Println("\n➡ Importing scenes...")
+		if err := importAllScenesCmd.RunE(importAllScenesCmd, []string{}); err != nil {
+			return fmt.Errorf("scene import failed: %w", err)
+		}
+
+		fmt.Println("\n✓ Completed TPDB bulk import for performers, studios, and scenes")
+		return nil
+	},
+}
+
 func init() {
 	importCmd.AddCommand(importPerformerCmd)
 	importCmd.AddCommand(importAllPerformersCmd)
@@ -67,6 +128,7 @@ func init() {
 	importCmd.AddCommand(importSceneCmd)
 	importCmd.AddCommand(importAllScenesCmd)
 	importCmd.AddCommand(importMovieCmd)
+	importCmd.AddCommand(importAllCmd)
 
 	// Flags for bulk import
 	importAllPerformersCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)")
@@ -75,6 +137,12 @@ func init() {
 	importAllStudiosCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)")
 	importAllScenesCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)")
 	importAllScenesCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)")
+	importAllCmd.Flags().Int("performer-start-page", 1, "Performer import start page (for resuming)")
+	importAllCmd.Flags().Int("performer-max-pages", 0, "Maximum performer pages to import (0 = all)")
+	importAllCmd.Flags().Int("studio-start-page", 1, "Studio import start page (for resuming)")
+	importAllCmd.Flags().Int("studio-max-pages", 0, "Maximum studio pages to import (0 = all)")
+	importAllCmd.Flags().Int("scene-start-page", 1, "Scene import start page (for resuming)")
+	importAllCmd.Flags().Int("scene-max-pages", 0, "Maximum scene pages to import (0 = all)")
 
 	// Movie import flags
 	importMovieCmd.Flags().String("source", "adultemp", "Source to import from (adultemp)")
@@ -114,9 +182,9 @@ var syncAllCmd = &cobra.Command{
 		force, _ := cmd.Flags().GetBool("force")
 		interval, _ := cmd.Flags().GetDuration("interval")
 
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		database, err := getDB()
@@ -173,9 +241,9 @@ var syncPerformersCmd = &cobra.Command{
 		force, _ := cmd.Flags().GetBool("force")
 		interval, _ := cmd.Flags().GetDuration("interval")
 
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		database, err := getDB()
@@ -222,9 +290,9 @@ var syncStudiosCmd = &cobra.Command{
 		force, _ := cmd.Flags().GetBool("force")
 		interval, _ := cmd.Flags().GetDuration("interval")
 
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		database, err := getDB()
@@ -271,9 +339,9 @@ var syncScenesCmd = &cobra.Command{
 		force, _ := cmd.Flags().GetBool("force")
 		interval, _ := cmd.Flags().GetDuration("interval")
 
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		database, err := getDB()
@@ -376,6 +444,18 @@ func formatDuration(d time.Duration) string {
 	return fmt.Sprintf("%d days", int(d.Hours()/24))
 }
 
+func getTPDBAPIKey() (string, error) {
+	apiKey := os.Getenv(tpdbAPIKeyEnvVar)
+	if apiKey == "" {
+		return "", fmt.Errorf("%s environment variable is not set.\n%s", tpdbAPIKeyEnvVar, apiKeySetupInstructions())
+	}
+	return apiKey, nil
+}
+
+func apiKeySetupInstructions() string {
+	return fmt.Sprintf("Set it by running:\n\n  export %s=\"your-api-key\"", tpdbAPIKeyEnvVar)
+}
+
 func main() {
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -417,9 +497,9 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version information",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("Goondex v0.1.0-dev4")
+		fmt.Println("Goondex v0.1.0-dev5")
 		fmt.Println("Features:")
-		fmt.Println("  • TPDB integration with auto-import")
+		fmt.Println("  • TPDB integration with auto-import & bulk commands")
 		fmt.Println("  • Adult Empire scraper (scenes & performers)")
 		fmt.Println("  • Multi-source data merging")
 		fmt.Println("  • Grid-based web UI with GX components")
@@ -451,10 +531,9 @@ var performerSearchCmd = &cobra.Command{
 		if len(performers) == 0 {
 			fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
 
-			apiKey := os.Getenv("TPDB_API_KEY")
+			apiKey, err := getTPDBAPIKey()
-			if apiKey == "" {
+			if err != nil {
-				fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
+				fmt.Printf("⚠ %s\n", err)
-				fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
 				return nil
 			}
 
@@ -638,8 +717,10 @@ var performerUpdateCmd = &cobra.Command{
 
 		// Update from TPDB if available
 		if performer.Source == "tpdb" && performer.SourceID != "" {
-			apiKey := os.Getenv("TPDB_API_KEY")
+			apiKey, err := getTPDBAPIKey()
-			if apiKey != "" {
+			if err != nil {
+				fmt.Printf("⚠ %s\n", err)
+			} else {
 				fmt.Println("📥 Fetching latest data from TPDB...")
 				scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
 
@@ -657,8 +738,6 @@ var performerUpdateCmd = &cobra.Command{
 						performer = updatedPerformer
 					}
 				}
-			} else {
-				fmt.Println("⚠ TPDB_API_KEY not set, skipping TPDB update")
 			}
 		}
 
@@ -858,10 +937,9 @@ var studioSearchCmd = &cobra.Command{
 		if len(studios) == 0 {
 			fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
 
-			apiKey := os.Getenv("TPDB_API_KEY")
+			apiKey, err := getTPDBAPIKey()
-			if apiKey == "" {
+			if err != nil {
-				fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
+				fmt.Printf("⚠ %s\n", err)
-				fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
 				return nil
 			}
 
@@ -1022,10 +1100,9 @@ var sceneSearchCmd = &cobra.Command{
 		if len(scenes) == 0 {
 			fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
 
-			apiKey := os.Getenv("TPDB_API_KEY")
+			apiKey, err := getTPDBAPIKey()
-			if apiKey == "" {
+			if err != nil {
-				fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
+				fmt.Printf("⚠ %s\n", err)
-				fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
 				return nil
 			}
 
@@ -1224,9 +1301,9 @@ var importAllPerformersCmd = &cobra.Command{
 		maxPages, _ := cmd.Flags().GetInt("max-pages")
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -1336,9 +1413,9 @@ var importAllStudiosCmd = &cobra.Command{
 		maxPages, _ := cmd.Flags().GetInt("max-pages")
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -1454,9 +1531,9 @@ var importAllScenesCmd = &cobra.Command{
 		maxPages, _ := cmd.Flags().GetInt("max-pages")
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -1612,9 +1689,9 @@ var importPerformerCmd = &cobra.Command{
 		query := args[0]
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -1668,9 +1745,9 @@ var importStudioCmd = &cobra.Command{
 		query := args[0]
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -2166,9 +2243,9 @@ var importSceneCmd = &cobra.Command{
 		query := args[0]
 
 		// Get API key from environment
-		apiKey := os.Getenv("TPDB_API_KEY")
+		apiKey, err := getTPDBAPIKey()
-		if apiKey == "" {
+		if err != nil {
-			return fmt.Errorf("TPDB_API_KEY environment variable is not set")
+			return err
 		}
 
 		// Create TPDB scraper
@@ -2272,7 +2349,7 @@ Movies can be imported by:
 2. Direct URL: ./goondex import movie https://www.adultdvdempire.com/...
 
 Note: For bulk movie import, movies are best imported through Adult Empire's catalog.`,
-	Args:  cobra.MinimumNArgs(1),
+	Args: cobra.MinimumNArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		input := args[0]
 

docs/ADULT_EMPIRE_SCRAPER.md

@@ -1,7 +1,7 @@
 # Adult Empire Scraper Integration
 
-**Version**: v0.1.0-dev4
+**Version**: v0.1.0-dev5
-**Last Updated**: 2025-11-16
+**Last Updated**: 2025-11-17
 
 ## Overview
 
@@ -316,6 +316,9 @@ To improve Adult Empire scraping:
 
 ## Version History
 
+- **v0.1.0-dev5** (2025-11-17): Documentation refresh for TPDB bulk-import release
+  - Updated version metadata and changelog references
+  - Clarified rebuild steps for the CLI additions
 - **v0.1.0-dev4** (2025-11-16): Initial Adult Empire scraper implementation
   - HTTP client with cookie support
   - XPath parsing utilities
@@ -325,5 +328,5 @@ To improve Adult Empire scraping:
 
 ---
 
-**Last Updated**: 2025-11-16
+**Last Updated**: 2025-11-17
 **Maintainer**: Goondex Team