// text-duckduckgo.go package main import ( "fmt" "log" "net/http" "net/url" "strings" "time" "github.com/PuerkitoBio/goquery" ) const ( resultsPerPage = 10 ) func getVQD(query string) (string, error) { queryURL := fmt.Sprintf("https://duckduckgo.com/?q=%s", url.QueryEscape(query)) resp, err := http.Get(queryURL) if err != nil { return "", fmt.Errorf("failed to fetch vqd: %v", err) } defer resp.Body.Close() doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return "", fmt.Errorf("loading HTML document: %v", err) } var vqd string doc.Find("script").Each(func(i int, s *goquery.Selection) { text := s.Text() if strings.Contains(text, "vqd=\"") { start := strings.Index(text, "vqd=\"") + 5 end := strings.Index(text[start:], "\"") vqd = text[start : start+end] } }) if vqd == "" { return "", fmt.Errorf("vqd not found") } return vqd, nil } func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) { var results []TextSearchResult client := &http.Client{Timeout: 10 * time.Second} vqd, err := getVQD(query) if err != nil { return nil, fmt.Errorf("failed to get vqd: %v", err) } searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s&kl=%s&safe=%s&s=%d&vqd=%s", url.QueryEscape(query), lang, safe, (page-1)*resultsPerPage, vqd) req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %v", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("making request: %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) } doc, err := goquery.NewDocumentFromReader(resp.Body) if err != nil { return nil, fmt.Errorf("loading HTML document: %v", err) } doc.Find(".result__body").Each(func(i int, s *goquery.Selection) { header := s.Find(".result__a").Text() description := s.Find(".result__snippet").Text() rawURL, exists := s.Find(".result__a").Attr("href") if exists { parsedURL, err := url.Parse(rawURL) if err == nil { queryParams := parsedURL.Query() uddg := queryParams.Get("uddg") if uddg != "" { result := TextSearchResult{ URL: uddg, Header: strings.TrimSpace(header), Description: strings.TrimSpace(description), Source: "DuckDuckGo", } results = append(results, result) if debugMode { log.Printf("Processed DuckDuckGo result: %+v\n", result) } } else { if debugMode { log.Printf("Missing 'uddg' parameter in URL: %s\n", rawURL) } } } else { if debugMode { log.Printf("Error parsing URL: %s, error: %v\n", rawURL, err) } } } else { if debugMode { log.Printf("Missing 'href' attribute in result anchor tag\n") } } }) if len(results) == 0 { if debugMode { log.Println("No results found from DuckDuckGo") } } return results, nil }