Search/text-duckduckgo.go

// text-duckduckgo.go
package main

import (
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
	var results []TextSearchResult
	searchURL := buildDuckDuckGoSearchURL(query, page)

	resp, err := http.Get(searchURL)
	if err != nil {
		return nil, fmt.Errorf("making request: %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("loading HTML document: %v", err)
	}

	doc.Find(".result__body").Each(func(i int, s *goquery.Selection) {
		header := s.Find(".result__a").Text()
		description := s.Find(".result__snippet").Text()
		rawURL, exists := s.Find(".result__a").Attr("href")
		if exists {
			parsedURL, err := url.Parse(rawURL)
			if err == nil {
				queryParams := parsedURL.Query()
				uddg := queryParams.Get("uddg")
				if uddg != "" {
					result := TextSearchResult{
						URL:         uddg,
						Header:      strings.TrimSpace(header),
						Description: strings.TrimSpace(description),
					}
					results = append(results, result)
					if debugMode {
						log.Printf("Processed DuckDuckGo result: %+v\n", result)
					}
				}
			}
		}
	})

	return results, nil
}

func buildDuckDuckGoSearchURL(query string, page int) string {
	startParam := ""
	if page > 1 {
		startParam = fmt.Sprintf("&s=%d", (page-1)*10)
	}
	return fmt.Sprintf("https://duckduckgo.com/html/?q=%s%s", url.QueryEscape(query), startParam)
}