package main import ( "flag" "fmt" "html/template" "log" "net/http" "sort" "sync" "time" ) var ( debugMode bool resultsCache = NewResultsCache(6 * time.Hour) // Cache with 6-hour expiration ) func init() { flag.BoolVar(&debugMode, "debug", false, "enable debug mode") flag.Parse() } func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int) { startTime := time.Now() const resultsPerPage = 10 cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang} combinedResults := getResultsFromCacheOrFetch(cacheKey, query, safe, lang, page, resultsPerPage) hasPrevPage := page > 1 hasNextPage := len(combinedResults) == resultsPerPage displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage) // Always check and cache the next page if not enough results if hasNextPage { go cacheNextPageIfNotCached(query, safe, lang, page+1, resultsPerPage) } } func getResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page, resultsPerPage int) []TextSearchResult { cacheChan := make(chan []TextSearchResult) var combinedResults []TextSearchResult go func() { results, exists := resultsCache.Get(cacheKey) if exists { log.Println("Cache hit") cacheChan <- results } else { log.Println("Cache miss") cacheChan <- nil } }() select { case combinedResults = <-cacheChan: if combinedResults == nil { combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage) resultsCache.Set(cacheKey, combinedResults) } case <-time.After(2 * time.Second): log.Println("Cache check timeout") combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage) resultsCache.Set(cacheKey, combinedResults) } return combinedResults } func cacheNextPageIfNotCached(query, safe, lang string, page, resultsPerPage int) { cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang} if _, exists := resultsCache.Get(cacheKey); !exists { log.Printf("Next page %d not cached, caching now...", page) nextPageResults := fetchResultsUntilFull(query, safe, lang, page, resultsPerPage) resultsCache.Set(cacheKey, nextPageResults) } else { log.Printf("Next page %d already cached", page) } } func fetchResultsUntilFull(query, safe, lang string, targetPage, resultsPerPage int) []TextSearchResult { var combinedResults []TextSearchResult currentPage := 1 resultsNeeded := targetPage * resultsPerPage for len(combinedResults) < resultsNeeded { cacheKey := CacheKey{Query: query, Page: currentPage, Safe: safe, Lang: lang} cachedResults, exists := resultsCache.Get(cacheKey) if exists { combinedResults = append(combinedResults, cachedResults...) } else { results := fetchAndCacheResults(query, safe, lang, currentPage, resultsPerPage) if len(results) == 0 { break } combinedResults = append(combinedResults, results...) resultsCache.Set(cacheKey, results) } currentPage++ // Stop fetching if we have enough results for the target page and the next page if len(combinedResults) >= resultsNeeded+resultsPerPage { break } } startIndex := (targetPage - 1) * resultsPerPage endIndex := startIndex + resultsPerPage if startIndex >= len(combinedResults) { return []TextSearchResult{} } if endIndex > len(combinedResults) { endIndex = len(combinedResults) } return combinedResults[startIndex:endIndex] } func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult { var combinedResults []TextSearchResult var wg sync.WaitGroup var mu sync.Mutex resultsChan := make(chan []TextSearchResult) searchFuncs := []struct { Func func(string, string, string, int) ([]TextSearchResult, error) Source string }{ {PerformGoogleTextSearch, "Google"}, {PerformDuckDuckGoTextSearch, "DuckDuckGo"}, {PerformQwantTextSearch, "Qwant"}, } wg.Add(len(searchFuncs)) for _, searchFunc := range searchFuncs { go func(searchFunc func(string, string, string, int) ([]TextSearchResult, error), source string) { defer wg.Done() results, err := searchFunc(query, safe, lang, page) if err == nil { for i := range results { results[i].Source = source } resultsChan <- results } else { log.Printf("Error performing search from %s: %v", source, err) } }(searchFunc.Func, searchFunc.Source) } go func() { wg.Wait() close(resultsChan) }() for results := range resultsChan { mu.Lock() combinedResults = append(combinedResults, results...) mu.Unlock() } sort.SliceStable(combinedResults, func(i, j int) bool { return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source) }) log.Printf("Fetched %d results for page %d", len(combinedResults), page) return combinedResults } func paginateResults(results []TextSearchResult, page, resultsPerPage int) []TextSearchResult { startIndex := (page - 1) * resultsPerPage endIndex := startIndex + resultsPerPage log.Printf("Paginating results: startIndex=%d, endIndex=%d, totalResults=%d", startIndex, endIndex, len(results)) if startIndex >= len(results) { return []TextSearchResult{} } if endIndex > len(results) { endIndex = len(results) } return results[startIndex:endIndex] } func sourceOrder(source string) int { switch source { case "Google": return 1 case "DuckDuckGo": return 2 case "Qwant": return 3 default: return 4 } } func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64, page int, hasPrevPage, hasNextPage bool) { log.Printf("Displaying results for page %d", page) log.Printf("Total results: %d", len(results)) log.Printf("Has previous page: %t, Has next page: %t", hasPrevPage, hasNextPage) tmpl, err := template.New("text.html").Funcs(template.FuncMap{ "sub": func(a, b int) int { return a - b }, "add": func(a, b int) int { return a + b }, }).ParseFiles("templates/text.html") if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } data := struct { Results []TextSearchResult Query string Fetched string Page int HasPrevPage bool HasNextPage bool LanguageOptions []LanguageOption CurrentLang string }{ Results: results, Query: query, Fetched: fmt.Sprintf("%.2f seconds", elapsed), Page: page, HasPrevPage: hasPrevPage, HasNextPage: hasNextPage, LanguageOptions: languageOptions, CurrentLang: lang, } err = tmpl.Execute(w, data) if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) } }