package import_service import ( "context" "fmt" "log" "git.leaktechnologies.dev/stu/Goondex/internal/db" "git.leaktechnologies.dev/stu/Goondex/internal/model" "git.leaktechnologies.dev/stu/Goondex/internal/scraper/tpdb" ) // ProgressUpdate represents a progress update during import type ProgressUpdate struct { EntityType string `json:"entity_type"` Current int `json:"current"` Total int `json:"total"` Percent float64 `json:"percent"` Message string `json:"message"` } // ProgressCallback is called when progress is made type ProgressCallback func(update ProgressUpdate) // Service handles bulk import operations type Service struct { db *db.DB scraper *tpdb.Scraper } // NewService creates a new import service func NewService(database *db.DB, scraper *tpdb.Scraper) *Service { return &Service{ db: database, scraper: scraper, } } // ImportResult contains the results of an import operation type ImportResult struct { EntityType string Imported int Failed int Total int } // BulkImportAllPerformers imports all performers from TPDB func (s *Service) BulkImportAllPerformers(ctx context.Context) (*ImportResult, error) { return s.BulkImportAllPerformersWithProgress(ctx, nil) } // BulkImportAllPerformersWithProgress imports all performers from TPDB with progress updates func (s *Service) BulkImportAllPerformersWithProgress(ctx context.Context, progress ProgressCallback) (*ImportResult, error) { result := &ImportResult{ EntityType: "performers", } performerStore := db.NewPerformerStore(s.db) page := 1 for { performers, meta, err := s.scraper.ListPerformers(ctx, page) if err != nil { return result, fmt.Errorf("failed to fetch page %d: %w", page, err) } // Update total on first page if meta != nil && page == 1 { result.Total = meta.Total } // Import each performer for _, performer := range performers { if err := performerStore.Upsert(&performer); err != nil { log.Printf("Failed to import performer %s: %v", performer.Name, err) result.Failed++ } else { result.Imported++ } // Send progress update if progress != nil && result.Total > 0 { progress(ProgressUpdate{ EntityType: "performers", Current: result.Imported, Total: result.Total, Percent: float64(result.Imported) / float64(result.Total) * 100, Message: fmt.Sprintf("Imported %d/%d performers", result.Imported, result.Total), }) } } log.Printf("Imported page %d/%d of performers (%d/%d total)", page, meta.LastPage, result.Imported, result.Total) // Check if we've reached the last page if meta == nil || page >= meta.LastPage { break } page++ } return result, nil } // BulkImportAllStudios imports all studios from TPDB func (s *Service) BulkImportAllStudios(ctx context.Context) (*ImportResult, error) { return s.BulkImportAllStudiosWithProgress(ctx, nil) } // BulkImportAllStudiosWithProgress imports all studios from TPDB with progress updates func (s *Service) BulkImportAllStudiosWithProgress(ctx context.Context, progress ProgressCallback) (*ImportResult, error) { result := &ImportResult{ EntityType: "studios", } studioStore := db.NewStudioStore(s.db) page := 1 for { studios, meta, err := s.scraper.ListStudios(ctx, page) if err != nil { return result, fmt.Errorf("failed to fetch page %d: %w", page, err) } // Update total on first page if meta != nil && page == 1 { result.Total = meta.Total } // Import each studio for _, studio := range studios { if err := studioStore.Upsert(&studio); err != nil { log.Printf("Failed to import studio %s: %v", studio.Name, err) result.Failed++ } else { result.Imported++ } // Send progress update if progress != nil && result.Total > 0 { progress(ProgressUpdate{ EntityType: "studios", Current: result.Imported, Total: result.Total, Percent: float64(result.Imported) / float64(result.Total) * 100, Message: fmt.Sprintf("Imported %d/%d studios", result.Imported, result.Total), }) } } log.Printf("Imported page %d/%d of studios (%d/%d total)", page, meta.LastPage, result.Imported, result.Total) // Check if we've reached the last page if meta == nil || page >= meta.LastPage { break } page++ } return result, nil } // BulkImportAllScenes imports all scenes from TPDB func (s *Service) BulkImportAllScenes(ctx context.Context) (*ImportResult, error) { return s.BulkImportAllScenesWithProgress(ctx, nil) } // BulkImportAllScenesWithProgress imports all scenes from TPDB with progress updates func (s *Service) BulkImportAllScenesWithProgress(ctx context.Context, progress ProgressCallback) (*ImportResult, error) { result := &ImportResult{ EntityType: "scenes", } performerStore := db.NewPerformerStore(s.db) studioStore := db.NewStudioStore(s.db) sceneStore := db.NewSceneStore(s.db) tagStore := db.NewTagStore(s.db) page := 1 for { scenes, meta, err := s.scraper.ListScenes(ctx, page) if err != nil { return result, fmt.Errorf("failed to fetch page %d: %w", page, err) } // Update total on first page if meta != nil && page == 1 { result.Total = meta.Total } // Import each scene with its performers and tags for _, scene := range scenes { // First import performers from the scene for _, performer := range scene.Performers { if err := performerStore.Upsert(&performer); err != nil { log.Printf("Failed to import performer %s for scene %s: %v", performer.Name, scene.Title, err) } } // Import studio if present if scene.Studio != nil { if err := studioStore.Upsert(scene.Studio); err != nil { log.Printf("Failed to import studio %s for scene %s: %v", scene.Studio.Name, scene.Title, err) } // Look up the studio ID existingStudio, err := studioStore.GetBySourceID("tpdb", scene.Studio.SourceID) if err == nil && existingStudio != nil { scene.StudioID = &existingStudio.ID } } // Import tags for _, tag := range scene.Tags { if err := tagStore.Upsert(&tag); err != nil { log.Printf("Failed to import tag %s for scene %s: %v", tag.Name, scene.Title, err) } } // Import the scene if err := sceneStore.Upsert(&scene); err != nil { log.Printf("Failed to import scene %s: %v", scene.Title, err) result.Failed++ continue } // Get the scene ID existingScene, err := sceneStore.GetBySourceID("tpdb", scene.SourceID) if err != nil { log.Printf("Failed to lookup scene %s after import: %v", scene.Title, err) result.Failed++ continue } // Link performers to scene for _, performer := range scene.Performers { existingPerformer, err := performerStore.GetBySourceID("tpdb", performer.SourceID) if err == nil && existingPerformer != nil { if err := sceneStore.AddPerformer(existingScene.ID, existingPerformer.ID); err != nil { log.Printf("Failed to link performer %s to scene %s: %v", performer.Name, scene.Title, err) } } } // Link tags to scene for _, tag := range scene.Tags { existingTag, err := tagStore.GetBySourceID("tpdb", tag.SourceID) if err == nil && existingTag != nil { if err := sceneStore.AddTag(existingScene.ID, existingTag.ID); err != nil { log.Printf("Failed to link tag %s to scene %s: %v", tag.Name, scene.Title, err) } } } result.Imported++ // Send progress update if progress != nil && result.Total > 0 { progress(ProgressUpdate{ EntityType: "scenes", Current: result.Imported, Total: result.Total, Percent: float64(result.Imported) / float64(result.Total) * 100, Message: fmt.Sprintf("Imported %d/%d scenes", result.Imported, result.Total), }) } } log.Printf("Imported page %d/%d of scenes (%d/%d total)", page, meta.LastPage, result.Imported, result.Total) // Check if we've reached the last page if meta == nil || page >= meta.LastPage { break } page++ } return result, nil } // BulkImportAll imports all data from TPDB (performers, studios, scenes) func (s *Service) BulkImportAll(ctx context.Context) ([]ImportResult, error) { var results []ImportResult log.Println("Starting bulk import of all TPDB data...") // Import performers first log.Println("Importing performers...") performerResult, err := s.BulkImportAllPerformers(ctx) if err != nil { return results, fmt.Errorf("failed to import performers: %w", err) } results = append(results, *performerResult) // Import studios log.Println("Importing studios...") studioResult, err := s.BulkImportAllStudios(ctx) if err != nil { return results, fmt.Errorf("failed to import studios: %w", err) } results = append(results, *studioResult) // Import scenes (with their performers and tags) log.Println("Importing scenes...") sceneResult, err := s.BulkImportAllScenes(ctx) if err != nil { return results, fmt.Errorf("failed to import scenes: %w", err) } results = append(results, *sceneResult) log.Println("Bulk import complete!") return results, nil } // ImportScene imports a single scene with all its related data func (s *Service) ImportScene(ctx context.Context, scene *model.Scene) error { performerStore := db.NewPerformerStore(s.db) studioStore := db.NewStudioStore(s.db) sceneStore := db.NewSceneStore(s.db) tagStore := db.NewTagStore(s.db) // Import performers first for _, performer := range scene.Performers { if err := performerStore.Upsert(&performer); err != nil { return fmt.Errorf("failed to import performer %s: %w", performer.Name, err) } } // Import tags for _, tag := range scene.Tags { if err := tagStore.Upsert(&tag); err != nil { return fmt.Errorf("failed to import tag %s: %w", tag.Name, err) } } // Import studio if present if scene.Studio != nil { if err := studioStore.Upsert(scene.Studio); err != nil { return fmt.Errorf("failed to import studio %s: %w", scene.Studio.Name, err) } // Look up the studio ID existingStudio, err := studioStore.GetBySourceID("tpdb", scene.Studio.SourceID) if err == nil && existingStudio != nil { scene.StudioID = &existingStudio.ID } } // Import the scene if err := sceneStore.Upsert(scene); err != nil { return fmt.Errorf("failed to import scene: %w", err) } // Get the scene ID existingScene, err := sceneStore.GetBySourceID("tpdb", scene.SourceID) if err != nil { return fmt.Errorf("failed to lookup scene after import: %w", err) } // Link performers to scene for _, performer := range scene.Performers { existingPerformer, err := performerStore.GetBySourceID("tpdb", performer.SourceID) if err == nil && existingPerformer != nil { if err := sceneStore.AddPerformer(existingScene.ID, existingPerformer.ID); err != nil { return fmt.Errorf("failed to link performer %s: %w", performer.Name, err) } } } // Link tags to scene for _, tag := range scene.Tags { existingTag, err := tagStore.GetBySourceID("tpdb", tag.SourceID) if err == nil && existingTag != nil { if err := sceneStore.AddTag(existingScene.ID, existingTag.ID); err != nil { return fmt.Errorf("failed to link tag %s: %w", tag.Name, err) } } } return nil }