Search/text.go

package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"time"
)

var textSearchEngines []SearchEngine
var textResultsChan = make(chan []TextSearchResult)

func init() {
	textSearchEngines = []SearchEngine{
		{Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch), Weight: 1},
		{Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch), Weight: 2},
		{Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch), Weight: 2},
		{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch), Weight: 5},
		// {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXNGTextSearch), Weight: 2}, // Uncomment when implemented
	}
}

func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int) {
	startTime := time.Now()

	cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
	combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, safe, lang, page)

	hasPrevPage := page > 1 // dupe

	//displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)

	// Prefetch next and previous pages
	go prefetchPage(query, safe, lang, page+1)
	if hasPrevPage {
		go prefetchPage(query, safe, lang, page-1)
	}

	elapsedTime := time.Since(startTime)
	tmpl, err := template.New("text.html").Funcs(funcs).ParseFiles("templates/text.html")
	if err != nil {
		log.Printf("Error parsing template: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		return
	}

	data := struct {
		Results         []TextSearchResult
		Query           string
		Page            int
		Fetched         string
		LanguageOptions []LanguageOption
		CurrentLang     string
		HasPrevPage     bool
		HasNextPage     bool
		NoResults       bool
	}{
		Results:         combinedResults,
		Query:           query,
		Page:            page,
		Fetched:         fmt.Sprintf("%.2f seconds", elapsedTime.Seconds()),
		LanguageOptions: languageOptions,
		CurrentLang:     lang,
		HasPrevPage:     page > 1,
		HasNextPage:     len(combinedResults) >= 50,
		NoResults:       len(combinedResults) == 0,
	}

	err = tmpl.Execute(w, data)
	if err != nil {
		log.Printf("Error executing template: %v", err)
		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
	}
}

func getTextResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page int) []TextSearchResult {
	cacheChan := make(chan []SearchResult)
	var combinedResults []TextSearchResult

	go func() {
		results, exists := resultsCache.Get(cacheKey)
		if exists {
			log.Println("Cache hit")
			cacheChan <- results
		} else {
			log.Println("Cache miss")
			cacheChan <- nil
		}
	}()

	select {
	case results := <-cacheChan:
		if results == nil {
			combinedResults = fetchTextResults(query, safe, lang, page)
			if len(combinedResults) > 0 {
				resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
			}
		} else {
			textResults, _, _ := convertToSpecificResults(results)
			combinedResults = textResults
		}
	case <-time.After(2 * time.Second):
		log.Println("Cache check timeout")
		combinedResults = fetchTextResults(query, safe, lang, page)
		if len(combinedResults) > 0 {
			resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
		}
	}

	return combinedResults
}

func prefetchPage(query, safe, lang string, page int) {
	cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
	if _, exists := resultsCache.Get(cacheKey); !exists {
		log.Printf("Page %d not cached, caching now...", page)
		pageResults := fetchTextResults(query, safe, lang, page)
		if len(pageResults) > 0 {
			resultsCache.Set(cacheKey, convertToSearchResults(pageResults))
		}
	} else {
		log.Printf("Page %d already cached", page)
	}
}

func fetchTextResults(query, safe, lang string, page int) []TextSearchResult {
	var results []TextSearchResult

	for _, engine := range textSearchEngines {
		log.Printf("Using search engine: %s", engine.Name)

		searchResults, duration, err := engine.Func(query, safe, lang, page)
		updateEngineMetrics(&engine, duration, err == nil)
		if err != nil {
			log.Printf("Error performing search with %s: %v", engine.Name, err)
			continue
		}

		results = append(results, validateResults(searchResults)...)

		// If results are found, break out of the loop
		if len(results) > 0 {
			break
		}
	}

	// If no results found after trying all engines
	if len(results) == 0 {
		log.Printf("No text results found for query: %s, trying other nodes", query)
		results = tryOtherNodesForTextSearch(query, safe, lang, page, []string{hostID})
	}

	return results
}

func validateResults(searchResults []SearchResult) []TextSearchResult {
	var validResults []TextSearchResult

	// Remove anything that is missing a URL or Header
	for _, result := range searchResults {
		textResult := result.(TextSearchResult)
		if textResult.URL != "" || textResult.Header != "" {
			validResults = append(validResults, textResult)
		}
	}

	return validResults
}

func wrapTextSearchFunc(f func(string, string, string, int) ([]TextSearchResult, time.Duration, error)) func(string, string, string, int) ([]SearchResult, time.Duration, error) {
	return func(query, safe, lang string, page int) ([]SearchResult, time.Duration, error) {
		textResults, duration, err := f(query, safe, lang, page)
		if err != nil {
			return nil, duration, err
		}
		searchResults := make([]SearchResult, len(textResults))
		for i, result := range textResults {
			searchResults[i] = result
		}
		return searchResults, duration, nil
	}
}

func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes []string) []TextSearchResult {
	for _, nodeAddr := range peers {
		if contains(visitedNodes, nodeAddr) {
			continue // Skip nodes already visited
		}
		results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
		if err != nil {
			log.Printf("Error contacting node %s: %v", nodeAddr, err)
			continue
		}
		if len(results) > 0 {
			return results
		}
	}
	return nil
}

func sendTextSearchRequestToNode(nodeAddr, query, safe, lang string, page int, visitedNodes []string) ([]TextSearchResult, error) {
	visitedNodes = append(visitedNodes, nodeAddr)
	searchParams := struct {
		Query        string   `json:"query"`
		Safe         string   `json:"safe"`
		Lang         string   `json:"lang"`
		Page         int      `json:"page"`
		ResponseAddr string   `json:"responseAddr"`
		VisitedNodes []string `json:"visitedNodes"`
	}{
		Query:        query,
		Safe:         safe,
		Lang:         lang,
		Page:         page,
		ResponseAddr: fmt.Sprintf("http://localhost:%d/node", config.Port),
		VisitedNodes: visitedNodes,
	}

	msgBytes, err := json.Marshal(searchParams)
	if err != nil {
		return nil, fmt.Errorf("failed to marshal search parameters: %v", err)
	}

	msg := Message{
		ID:      hostID,
		Type:    "search-text",
		Content: string(msgBytes),
	}

	err = sendMessage(nodeAddr, msg)
	if err != nil {
		return nil, fmt.Errorf("failed to send search request to node %s: %v", nodeAddr, err)
	}

	// Wait for results
	select {
	case res := <-textResultsChan:
		return res, nil
	case <-time.After(20 * time.Second):
		return nil, fmt.Errorf("timeout waiting for results from node %s", nodeAddr)
	}
}

func handleTextResultsMessage(msg Message) {
	var results []TextSearchResult
	err := json.Unmarshal([]byte(msg.Content), &results)
	if err != nil {
		log.Printf("Error unmarshalling text results: %v", err)
		return
	}

	log.Printf("Received text results: %+v", results)
	// Send results to textResultsChan
	go func() {
		textResultsChan <- results
	}()
}