Team_Goon/Goondex
Team_Goon/Goondex
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare v0.1.0-dev5

Browse Source
This commit is contained in:
Stu Leak 2025-11-17 14:38:58 -05:00
parent 16fb407a3c
commit 324095c9e9
4 changed files with 163 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@ -1,5 +1,17 @@
# Changelog
## v0.1.0-dev5 (2025-11-17)
### 🧰 CLI & Import Improvements
- Added `goondex import all` plus explicit `all-performers`, `all-studios`, and `all-scenes` bulk commands so the entire TPDB catalog can be ingested with one invocation.
- Each bulk import now exposes start/max page flags even when running the combined command, making resume workflows painless.
- Standardized TPDB API key handling across commands; errors now include clear export instructions via the shared helper.
### 📖 Developer Experience
- `README.md` now documents when/how to rebuild the CLI binary, preventing stale builds after source changes.
- `docs/ADULT_EMPIRE_SCRAPER.md` metadata bumped to v0.1.0-dev5.
- `goondex version` output updated to reflect the new release and highlight the bulk import capability.
## v0.1.0-dev4 (2025-11-16)
### 🎨 Web UI Enhancements

18
README.md
View File

@ -160,6 +160,24 @@ go build -o bin/goondex ./cmd/goondex
go run ./cmd/goondex performer-search "test"
```
### Building & Rebuilding the CLI
The Goondex binary is not rebuilt automatically—whenever you change Go files (especially under `cmd/goondex` or `internal/*`), rebuild before re-running commands.
```bash
# Clean out any previous binary (prevents running stale code)
rm -f goondex bin/goondex
# Build the latest CLI
go build -o goondex ./cmd/goondex
```
After rebuilding, rerun `./goondex` (or the binary under `bin/`) so new commands like `import all` become available. Repeat the build whenever:
- You pull new commits (e.g., moving to v0.1.0-dev5)
- CLI command definitions change
- Shared packages under `internal/` are modified
- You switch Go versions or modules are updated (`go mod tidy`, `go get`, etc.)
## Contributing
This is a personal project, but contributions are welcome! Please open an issue before submitting large changes.

177
cmd/goondex/main.go
View File

@ -4,10 +4,10 @@ import (
"context"
"fmt"
"os"
"strconv"
"time"
"github.com/spf13/cobra"
"git.leaktechnologies.dev/stu/Goondex/internal/db"
"git.leaktechnologies.dev/stu/Goondex/internal/model"
"git.leaktechnologies.dev/stu/Goondex/internal/scraper/adultemp"
@ -15,8 +15,11 @@ import (
"git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb"
"git.leaktechnologies.dev/stu/Goondex/internal/sync"
"git.leaktechnologies.dev/stu/Goondex/internal/web"
"github.com/spf13/cobra"
)
const tpdbAPIKeyEnvVar = "TPDB_API_KEY"
var (
dbPath string
rootCmd = &cobra.Command{
@ -59,6 +62,64 @@ var importCmd = &cobra.Command{
Long: `Import performers, studios, and scenes from ThePornDB into your local database.`,
}
var importAllCmd = &cobra.Command{
Use: "all",
Short: "Import all TPDB performers, studios, and scenes",
Long: `Run the performer, studio, and scene bulk imports sequentially to ingest the entire TPDB catalog.
Use the per-entity pagination flags to resume or limit parts of the import.`,
RunE: func(cmd *cobra.Command, args []string) error {
performerStart, _ := cmd.Flags().GetInt("performer-start-page")
performerMax, _ := cmd.Flags().GetInt("performer-max-pages")
studioStart, _ := cmd.Flags().GetInt("studio-start-page")
studioMax, _ := cmd.Flags().GetInt("studio-max-pages")
sceneStart, _ := cmd.Flags().GetInt("scene-start-page")
sceneMax, _ := cmd.Flags().GetInt("scene-max-pages")
// Propagate flag values to the existing per-entity commands
if err := importAllPerformersCmd.Flags().Set("start-page", strconv.Itoa(performerStart)); err != nil {
return fmt.Errorf("failed to set performer start page: %w", err)
}
if err := importAllPerformersCmd.Flags().Set("max-pages", strconv.Itoa(performerMax)); err != nil {
return fmt.Errorf("failed to set performer max pages: %w", err)
}
if err := importAllStudiosCmd.Flags().Set("start-page", strconv.Itoa(studioStart)); err != nil {
return fmt.Errorf("failed to set studio start page: %w", err)
}
if err := importAllStudiosCmd.Flags().Set("max-pages", strconv.Itoa(studioMax)); err != nil {
return fmt.Errorf("failed to set studio max pages: %w", err)
}
if err := importAllScenesCmd.Flags().Set("start-page", strconv.Itoa(sceneStart)); err != nil {
return fmt.Errorf("failed to set scene start page: %w", err)
}
if err := importAllScenesCmd.Flags().Set("max-pages", strconv.Itoa(sceneMax)); err != nil {
return fmt.Errorf("failed to set scene max pages: %w", err)
}
fmt.Println("╔═══════════════════════════════════════════════════════════════╗")
fmt.Println("║ TPDB BULK IMPORT - ALL ENTITIES ║")
fmt.Println("╚═══════════════════════════════════════════════════════════════╝")
fmt.Println("\n➡ Importing performers...")
if err := importAllPerformersCmd.RunE(importAllPerformersCmd, []string{}); err != nil {
return fmt.Errorf("performer import failed: %w", err)
}
fmt.Println("\n➡ Importing studios...")
if err := importAllStudiosCmd.RunE(importAllStudiosCmd, []string{}); err != nil {
return fmt.Errorf("studio import failed: %w", err)
}
fmt.Println("\n➡ Importing scenes...")
if err := importAllScenesCmd.RunE(importAllScenesCmd, []string{}); err != nil {
return fmt.Errorf("scene import failed: %w", err)
}
fmt.Println("\n✓ Completed TPDB bulk import for performers, studios, and scenes")
return nil
},
}
func init() {
importCmd.AddCommand(importPerformerCmd)
importCmd.AddCommand(importAllPerformersCmd)
@ -67,6 +128,7 @@ func init() {
importCmd.AddCommand(importSceneCmd)
importCmd.AddCommand(importAllScenesCmd)
importCmd.AddCommand(importMovieCmd)
importCmd.AddCommand(importAllCmd)
// Flags for bulk import
importAllPerformersCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)")
@ -75,6 +137,12 @@ func init() {
importAllStudiosCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)")
importAllScenesCmd.Flags().Int("start-page", 1, "Page to start from (for resuming)")
importAllScenesCmd.Flags().Int("max-pages", 0, "Maximum pages to import (0 = all)")
importAllCmd.Flags().Int("performer-start-page", 1, "Performer import start page (for resuming)")
importAllCmd.Flags().Int("performer-max-pages", 0, "Maximum performer pages to import (0 = all)")
importAllCmd.Flags().Int("studio-start-page", 1, "Studio import start page (for resuming)")
importAllCmd.Flags().Int("studio-max-pages", 0, "Maximum studio pages to import (0 = all)")
importAllCmd.Flags().Int("scene-start-page", 1, "Scene import start page (for resuming)")
importAllCmd.Flags().Int("scene-max-pages", 0, "Maximum scene pages to import (0 = all)")
// Movie import flags
importMovieCmd.Flags().String("source", "adultemp", "Source to import from (adultemp)")
@ -114,9 +182,9 @@ var syncAllCmd = &cobra.Command{
force, _ := cmd.Flags().GetBool("force")
interval, _ := cmd.Flags().GetDuration("interval")
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
database, err := getDB()
@ -173,9 +241,9 @@ var syncPerformersCmd = &cobra.Command{
force, _ := cmd.Flags().GetBool("force")
interval, _ := cmd.Flags().GetDuration("interval")
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
database, err := getDB()
@ -222,9 +290,9 @@ var syncStudiosCmd = &cobra.Command{
force, _ := cmd.Flags().GetBool("force")
interval, _ := cmd.Flags().GetDuration("interval")
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
database, err := getDB()
@ -271,9 +339,9 @@ var syncScenesCmd = &cobra.Command{
force, _ := cmd.Flags().GetBool("force")
interval, _ := cmd.Flags().GetDuration("interval")
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
database, err := getDB()
@ -376,6 +444,18 @@ func formatDuration(d time.Duration) string {
return fmt.Sprintf("%d days", int(d.Hours()/24))
}
func getTPDBAPIKey() (string, error) {
apiKey := os.Getenv(tpdbAPIKeyEnvVar)
if apiKey == "" {
return "", fmt.Errorf("%s environment variable is not set.\n%s", tpdbAPIKeyEnvVar, apiKeySetupInstructions())
}
return apiKey, nil
}
func apiKeySetupInstructions() string {
return fmt.Sprintf("Set it by running:\n\n export %s=\"your-api-key\"", tpdbAPIKeyEnvVar)
}
func main() {
if err := rootCmd.Execute(); err != nil {
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@ -417,9 +497,9 @@ var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version information",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("Goondex v0.1.0-dev4")
fmt.Println("Goondex v0.1.0-dev5")
fmt.Println("Features:")
fmt.Println(" • TPDB integration with auto-import")
fmt.Println(" • TPDB integration with auto-import & bulk commands")
fmt.Println(" • Adult Empire scraper (scenes & performers)")
fmt.Println(" • Multi-source data merging")
fmt.Println(" • Grid-based web UI with GX components")
@ -451,10 +531,9 @@ var performerSearchCmd = &cobra.Command{
if len(performers) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
apiKey, err := getTPDBAPIKey()
if err != nil {
fmt.Printf("⚠ %s\n", err)
return nil
}
@ -638,8 +717,10 @@ var performerUpdateCmd = &cobra.Command{
// Update from TPDB if available
if performer.Source == "tpdb" && performer.SourceID != "" {
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey != "" {
apiKey, err := getTPDBAPIKey()
if err != nil {
fmt.Printf("⚠ %s\n", err)
} else {
fmt.Println("📥 Fetching latest data from TPDB...")
scraper := tpdb.NewScraper("https://api.theporndb.net", apiKey)
@ -657,8 +738,6 @@ var performerUpdateCmd = &cobra.Command{
performer = updatedPerformer
}
}
} else {
fmt.Println("⚠ TPDB_API_KEY not set, skipping TPDB update")
}
}
@ -858,10 +937,9 @@ var studioSearchCmd = &cobra.Command{
if len(studios) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
apiKey, err := getTPDBAPIKey()
if err != nil {
fmt.Printf("⚠ %s\n", err)
return nil
}
@ -1022,10 +1100,9 @@ var sceneSearchCmd = &cobra.Command{
if len(scenes) == 0 {
fmt.Printf("No local results found. Searching TPDB for '%s'...\n", query)
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
fmt.Println("⚠ TPDB_API_KEY not set. Cannot fetch from TPDB.")
fmt.Println("Set it with: export TPDB_API_KEY=\"your-key\"")
apiKey, err := getTPDBAPIKey()
if err != nil {
fmt.Printf("⚠ %s\n", err)
return nil
}
@ -1224,9 +1301,9 @@ var importAllPerformersCmd = &cobra.Command{
maxPages, _ := cmd.Flags().GetInt("max-pages")
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -1336,9 +1413,9 @@ var importAllStudiosCmd = &cobra.Command{
maxPages, _ := cmd.Flags().GetInt("max-pages")
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -1454,9 +1531,9 @@ var importAllScenesCmd = &cobra.Command{
maxPages, _ := cmd.Flags().GetInt("max-pages")
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -1612,9 +1689,9 @@ var importPerformerCmd = &cobra.Command{
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -1668,9 +1745,9 @@ var importStudioCmd = &cobra.Command{
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -2166,9 +2243,9 @@ var importSceneCmd = &cobra.Command{
query := args[0]
// Get API key from environment
apiKey := os.Getenv("TPDB_API_KEY")
if apiKey == "" {
return fmt.Errorf("TPDB_API_KEY environment variable is not set")
apiKey, err := getTPDBAPIKey()
if err != nil {
return err
}
// Create TPDB scraper
@ -2272,7 +2349,7 @@ Movies can be imported by:
2. Direct URL: ./goondex import movie https://www.adultdvdempire.com/...
Note: For bulk movie import, movies are best imported through Adult Empire's catalog.`,
Args: cobra.MinimumNArgs(1),
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
input := args[0]

9
docs/ADULT_EMPIRE_SCRAPER.md
View File

@ -1,7 +1,7 @@
# Adult Empire Scraper Integration
**Version**: v0.1.0-dev4
**Last Updated**: 2025-11-16
**Version**: v0.1.0-dev5
**Last Updated**: 2025-11-17
## Overview
@ -316,6 +316,9 @@ To improve Adult Empire scraping:
## Version History
- **v0.1.0-dev5** (2025-11-17): Documentation refresh for TPDB bulk-import release
- Updated version metadata and changelog references
- Clarified rebuild steps for the CLI additions
- **v0.1.0-dev4** (2025-11-16): Initial Adult Empire scraper implementation
- HTTP client with cookie support
- XPath parsing utilities
@ -325,5 +328,5 @@ To improve Adult Empire scraping:
---
**Last Updated**: 2025-11-16
**Last Updated**: 2025-11-17
**Maintainer**: Goondex Team