cache v1 + debug mode + buttons on text results

cache.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"sync"
+)
+
+// TextSearchResult represents a single search result item.
+type TextSearchResult struct {
+	URL         string
+	Header      string
+	Description string
+	Source      string
+}
+
+// CacheKey represents the key used to store search results in the cache.
+type CacheKey struct {
+	Query string
+	Page  int
+	Safe  string
+	Lang  string
+}
+
+// ResultsCache is a thread-safe map for caching search results by composite keys.
+type ResultsCache struct {
+	mu      sync.Mutex
+	results map[string][]TextSearchResult
+}
+
+// NewResultsCache creates a new ResultsCache.
+func NewResultsCache() *ResultsCache {
+	return &ResultsCache{
+		results: make(map[string][]TextSearchResult),
+	}
+}
+
+// Get retrieves the results for a given key from the cache.
+func (rc *ResultsCache) Get(key CacheKey) ([]TextSearchResult, bool) {
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+	results, exists := rc.results[rc.keyToString(key)]
+	return results, exists
+}
+
+// Set stores the results for a given key in the cache.
+func (rc *ResultsCache) Set(key CacheKey, results []TextSearchResult) {
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+	rc.results[rc.keyToString(key)] = results
+}
+
+// keyToString converts a CacheKey to a string representation.
+func (rc *ResultsCache) keyToString(key CacheKey) string {
+	return fmt.Sprintf("%s|%d|%s|%s", key.Query, key.Page, key.Safe, key.Lang)
+}

images.go

@@ -56,9 +56,9 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch
 		offset = (page - 1) * resultsPerPage
 	}
 
-	// Ensuring safe search is enabled by default if not specified
+	// Ensuring safe search is disabled by default if not specified
 	if safe == "" {
-		safe = "1"
+		safe = "0"
 	}
 
 	// Defaulting to English Canada locale if not specified
@@ -66,8 +66,7 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch
 		lang = "en_CA"
 	}
 
-	// Format &lang=lang_de is incorret, implement fix !
+	// Format &lang=lang_de is incorrect, implement fix !
-
 	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/images?t=images&q=%s&count=%d&locale=%s&offset=%d&device=desktop&tgp=2&safesearch=%s",
 		url.QueryEscape(query),
 		resultsPerPage,

main.go

@@ -1,3 +1,4 @@
+// main.go
 package main
 
 import (
@@ -88,6 +89,9 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
 		var err error
 		page, err = strconv.Atoi(pageStr)
 		if err != nil || page < 1 {
+			if debugMode {
+				log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
+			}
 			page = 1 // Default to page 1 if no valid page is specified
 		}
 	} else if r.Method == "POST" {
@@ -95,6 +99,15 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
 		safe = r.FormValue("safe")
 		lang = r.FormValue("lang")
 		searchType = r.FormValue("t")
+		pageStr := r.FormValue("p")
+		var err error
+		page, err = strconv.Atoi(pageStr)
+		if err != nil || page < 1 {
+			if debugMode {
+				log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
+			}
+			page = 1 // Default to page 1 if no valid page is specified
+		}
 	}
 
 	if query == "" {
@@ -104,7 +117,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
 
 	switch searchType {
 	case "text":
-		HandleTextSearch(w, query, safe, lang)
+		HandleTextSearch(w, query, safe, lang, page)
 	case "image":
 		handleImageSearch(w, query, safe, lang, page)
 	case "video":

run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go --debug
+go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go --debug

templates/text.html

@@ -42,7 +42,6 @@
                 <button name="t" value="torrent" class="clickable">Torrents</button>
             </div>
         </div>
-        </div>
     </form>
     <form class="results_settings" action="/search" method="get">
         <input type="hidden" name="q" value="{{ .Query }}">
@@ -58,7 +57,6 @@
         <button class="results-save" name="t" value="text">Apply settings</button>
     </form>    
     <div class="results">
-        <!-- Results go here -->
         {{if .Results}}
             {{range .Results}}
                 <div class="result_item">
@@ -72,7 +70,18 @@
             <div class="no-results">No results found for '{{ .Query }}'. Try different keywords.</div>
         {{end}}
     </div>
-
+    <div class="prev-next prev-img">
+        <form action="/search" method="get">
+            <input type="hidden" name="q" value="{{ .Query }}">
+            <input type="hidden" name="t" value="text">
+            {{ if .HasPrevPage }}
+            <button type="submit" name="p" value="{{ sub .Page 1 }}">Previous</button>
+            {{ end }}
+            {{ if .HasNextPage }}
+            <button type="submit" name="p" value="{{ add .Page 1 }}">Next</button>
+            {{ end }}
+        </form>
+    </div>
     <script>
         // Check if JavaScript is enabled and modify the DOM accordingly
         document.getElementById('content').classList.remove('js-enabled');

text-duckduckgo.go

@@ -7,15 +7,26 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
-func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
+func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
+	const resultsPerPage = 10
 	var results []TextSearchResult
-	searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s", url.QueryEscape(query))
 
-	resp, err := http.Get(searchURL)
+	client := &http.Client{Timeout: 10 * time.Second}
+	searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s&s=%d", url.QueryEscape(query), (page-1)*resultsPerPage)
+
+	req, err := http.NewRequest("GET", searchURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %v", err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("making request: %v", err)
 	}
@@ -49,10 +60,28 @@ func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult,
 					if debugMode {
 						log.Printf("Processed DuckDuckGo result: %+v\n", result)
 					}
+				} else {
+					if debugMode {
+						log.Printf("Missing 'uddg' parameter in URL: %s\n", rawURL)
 					}
 				}
+			} else {
+				if debugMode {
+					log.Printf("Error parsing URL: %s, error: %v\n", rawURL, err)
+				}
+			}
+		} else {
+			if debugMode {
+				log.Printf("Missing 'href' attribute in result anchor tag\n")
+			}
 		}
 	})
 
+	if len(results) == 0 {
+		if debugMode {
+			log.Println("No results found from DuckDuckGo")
+		}
+	}
+
 	return results, nil
 }

text-google.go

@@ -2,15 +2,18 @@
 package main
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
-func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
+func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
+	const resultsPerPage = 10
 	var results []TextSearchResult
 
 	client := &http.Client{}
@@ -24,29 +27,43 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
 		langParam = "&lr=" + lang
 	}
 
-	searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14"
+	// Calculate the start index based on the page number
+	startIndex := (page - 1) * resultsPerPage
+
+	searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14&start=" + strconv.Itoa(startIndex)
 
 	req, err := http.NewRequest("GET", searchURL, nil)
 	if err != nil {
-		log.Fatalf("Failed to create request: %v", err)
+		return nil, fmt.Errorf("failed to create request: %v", err)
 	}
 
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
 
 	resp, err := client.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("making request: %v", err)
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
 	doc, err := goquery.NewDocumentFromReader(resp.Body)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("loading HTML document: %v", err)
 	}
 
 	doc.Find(".yuRUbf").Each(func(i int, s *goquery.Selection) {
 		link := s.Find("a")
-		href, _ := link.Attr("href")
+		href, exists := link.Attr("href")
+		if !exists {
+			if debugMode {
+				log.Printf("No href attribute found for result %d\n", i)
+			}
+			return
+		}
+
 		header := link.Find("h3").Text()
 		header = strings.TrimSpace(strings.TrimSuffix(header, "›"))
 
@@ -67,5 +84,11 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
 		}
 	})
 
+	if len(results) == 0 {
+		if debugMode {
+			log.Println("No results found from Google")
+		}
+	}
+
 	return results, nil
 }

text-quant.go

@@ -1,8 +1,10 @@
+// text-qwant.go
 package main
 
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"net/http"
 	"net/url"
 	"time"
@@ -26,9 +28,11 @@ type QwantTextAPIResponse struct {
 }
 
 // PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
-func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
+func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
 	const resultsPerPage = 10
-	const offset = 0
+
+	// Calculate the offset based on the page number
+	offset := (page - 1) * resultsPerPage
 
 	// Ensure safe search is disabled by default if not specified
 	if safe == "" {
@@ -40,11 +44,12 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 		lang = "en_CA"
 	}
 
-	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
+	apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop&safesearch=%s",
 		url.QueryEscape(query),
 		resultsPerPage,
 		lang,
-		offset)
+		offset,
+		safe)
 
 	client := &http.Client{Timeout: 10 * time.Second}
 
@@ -93,6 +98,9 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
 func cleanQwantURL(rawURL string) string {
 	u, err := url.Parse(rawURL)
 	if err != nil {
+		if debugMode {
+			log.Printf("Error parsing URL: %v", err)
+		}
 		return rawURL
 	}
 	return u.Scheme + "://" + u.Host + u.Path

text.go

@@ -1,3 +1,4 @@
+// text.go
 package main
 
 import (
@@ -11,31 +12,49 @@ import (
 	"time"
 )
 
-type TextSearchResult struct {
+var (
-	URL         string
+	debugMode    bool
-	Header      string
+	resultsCache = NewResultsCache()
-	Description string
+)
-	Source      string
-}
-
-var debugMode bool
 
 func init() {
 	flag.BoolVar(&debugMode, "debug", false, "enable debug mode")
 	flag.Parse()
 }
 
-func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
+func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int) {
 	startTime := time.Now()
+	const resultsPerPage = 10
+
+	cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
+
+	// Try to get results from cache
+	combinedResults, exists := resultsCache.Get(cacheKey)
+	if !exists {
+		// Fetch results for the current page
+		combinedResults = fetchAndCacheResults(query, safe, lang, page, resultsPerPage)
+		resultsCache.Set(cacheKey, combinedResults)
+	}
+
+	// Pre-fetch and cache results for the next page
+	nextPageResults := fetchAndCacheResults(query, safe, lang, page+1, resultsPerPage)
+	resultsCache.Set(CacheKey{Query: query, Page: page + 1, Safe: safe, Lang: lang}, nextPageResults)
+
+	hasPrevPage := page > 1
+	hasNextPage := len(nextPageResults) > 0
+
+	displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
+}
+
+func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
 	var combinedResults []TextSearchResult
-	var resultMap = make(map[string]TextSearchResult)
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 
 	resultsChan := make(chan []TextSearchResult)
 
 	searchFuncs := []struct {
-		Func   func(string, string, string) ([]TextSearchResult, error)
+		Func   func(string, string, string, int) ([]TextSearchResult, error)
 		Source string
 	}{
 		{PerformGoogleTextSearch, "Google"},
@@ -46,9 +65,9 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
 	wg.Add(len(searchFuncs))
 
 	for _, searchFunc := range searchFuncs {
-		go func(searchFunc func(string, string, string) ([]TextSearchResult, error), source string) {
+		go func(searchFunc func(string, string, string, int) ([]TextSearchResult, error), source string) {
 			defer wg.Done()
-			results, err := searchFunc(query, safe, lang)
+			results, err := searchFunc(query, safe, lang, page)
 			if err == nil {
 				for i := range results {
 					results[i].Source = source
@@ -67,50 +86,52 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
 
 	for results := range resultsChan {
 		mu.Lock()
-		for _, result := range results {
+		combinedResults = append(combinedResults, results...)
-			existingResult, exists := resultMap[result.URL]
-			if !exists || shouldReplace(existingResult.Source, result.Source) {
-				resultMap[result.URL] = result
-			}
-			if debugMode {
-				log.Printf("Result from %s: %+v\n", result.Source, result)
-			}
-		}
 		mu.Unlock()
 	}
 
-	// Convert the map back to a slice
+	// Sort combinedResults by source priority: Google first, DuckDuckGo second, Qwant third
-	for _, result := range resultMap {
-		combinedResults = append(combinedResults, result)
-	}
-
-	// Custom sorting: Google first, DuckDuckGo second, Qwant third
 	sort.SliceStable(combinedResults, func(i, j int) bool {
 		return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
 	})
 
-	displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds())
+	// Paginate results
-}
+	startIndex := (page - 1) * resultsPerPage
+	endIndex := startIndex + resultsPerPage
 
-func shouldReplace(existingSource, newSource string) bool {
+	// Ensure startIndex and endIndex are within bounds
-	return sourceOrder(newSource) < sourceOrder(existingSource)
+	if startIndex >= len(combinedResults) {
+		return []TextSearchResult{}
+	}
+	if endIndex > len(combinedResults) {
+		endIndex = len(combinedResults)
+	}
+
+	return combinedResults[startIndex:endIndex]
 }
 
 func sourceOrder(source string) int {
 	switch source {
-	case "Qwant":
-		return 3
-	case "DuckDuckGo":
-		return 2
 	case "Google":
 		return 1
+	case "DuckDuckGo":
+		return 2
+	case "Qwant":
+		return 3
 	default:
 		return 4
 	}
 }
 
-func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64) {
+func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64, page int, hasPrevPage, hasNextPage bool) {
-	tmpl, err := template.ParseFiles("templates/text.html")
+	tmpl, err := template.New("text.html").Funcs(template.FuncMap{
+		"sub": func(a, b int) int {
+			return a - b
+		},
+		"add": func(a, b int) int {
+			return a + b
+		},
+	}).ParseFiles("templates/text.html")
 	if err != nil {
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
 		return
@@ -120,12 +141,18 @@ func displayResults(w http.ResponseWriter, results []TextSearchResult, query, la
 		Results         []TextSearchResult
 		Query           string
 		Fetched         string
+		Page            int
+		HasPrevPage     bool
+		HasNextPage     bool
 		LanguageOptions []LanguageOption
 		CurrentLang     string
 	}{
 		Results:         results,
 		Query:           query,
 		Fetched:         fmt.Sprintf("%.2f seconds", elapsed),
+		Page:            page,
+		HasPrevPage:     hasPrevPage,
+		HasNextPage:     hasNextPage,
 		LanguageOptions: languageOptions,
 		CurrentLang:     lang,
 	}