diff --git a/main.go b/main.go index 4e866af..6ced187 100644 --- a/main.go +++ b/main.go @@ -104,7 +104,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) { switch searchType { case "text": - handleTextSearch(w, query, safe, lang) + HandleTextSearch(w, query, safe, lang) case "image": handleImageSearch(w, query, safe, lang, page) case "video": diff --git a/run.sh b/run.sh index 59426bb..ede2f3e 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go \ No newline at end of file +go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go --debug \ No newline at end of file diff --git a/text-duckduckgo.go b/text-duckduckgo.go new file mode 100644 index 0000000..d003895 --- /dev/null +++ b/text-duckduckgo.go @@ -0,0 +1,58 @@ +// text-duckduckgo.go +package main + +import ( + "fmt" + "log" + "net/http" + "net/url" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult, error) { + var results []TextSearchResult + searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s", url.QueryEscape(query)) + + resp, err := http.Get(searchURL) + if err != nil { + return nil, fmt.Errorf("making request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("loading HTML document: %v", err) + } + + doc.Find(".result__body").Each(func(i int, s *goquery.Selection) { + header := s.Find(".result__a").Text() + description := s.Find(".result__snippet").Text() + rawURL, exists := s.Find(".result__a").Attr("href") + if exists { + parsedURL, err := url.Parse(rawURL) + if err == nil { + queryParams := parsedURL.Query() + uddg := queryParams.Get("uddg") + if uddg != "" { + result := TextSearchResult{ + URL: uddg, + Header: strings.TrimSpace(header), + Description: strings.TrimSpace(description), + } + results = append(results, result) + if debugMode { + log.Printf("Processed DuckDuckGo result: %+v\n", result) + } + } + } + } + }) + + return results, nil +} diff --git a/text-google.go b/text-google.go index b9b2b29..072d691 100644 --- a/text-google.go +++ b/text-google.go @@ -24,7 +24,7 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro langParam = "&lr=" + lang } - searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14" req, err := http.NewRequest("GET", searchURL, nil) if err != nil { @@ -56,11 +56,15 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro description = descSelection.Text() } - results = append(results, TextSearchResult{ + result := TextSearchResult{ URL: href, Header: header, Description: description, - }) + } + results = append(results, result) + if debugMode { + log.Printf("Google result: %+v\n", result) + } }) return results, nil diff --git a/text-quant.go b/text-quant.go index b697c1b..de8b03a 100644 --- a/text-quant.go +++ b/text-quant.go @@ -1,4 +1,3 @@ -// text-quant.go package main import ( @@ -13,22 +12,39 @@ import ( type QwantTextAPIResponse struct { Data struct { Result struct { - Items []struct { - Title string `json:"title"` - Url string `json:"url"` - Snippet string `json:"desc"` + Items struct { + Mainline []struct { + Items []struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"desc"` + } `json:"items"` + } `json:"mainline"` } `json:"items"` } `json:"result"` } `json:"data"` } +// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) { const resultsPerPage = 10 - apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?t=web&q=%s&count=%d&locale=%s&safesearch=%s", + const offset = 0 + + // Ensure safe search is disabled by default if not specified + if safe == "" { + safe = "0" + } + + // Default to English Canada locale if not specified + if lang == "" { + lang = "en_CA" + } + + apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop", url.QueryEscape(query), resultsPerPage, lang, - safe) + offset) client := &http.Client{Timeout: 10 * time.Second} @@ -54,14 +70,30 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error return nil, fmt.Errorf("decoding response: %v", err) } + // Extracting results from the nested JSON structure + if len(apiResp.Data.Result.Items.Mainline) == 0 { + return nil, fmt.Errorf("no search results found") + } + var results []TextSearchResult - for _, item := range apiResp.Data.Result.Items { + for _, item := range apiResp.Data.Result.Items.Mainline[0].Items { + cleanURL := cleanQwantURL(item.URL) results = append(results, TextSearchResult{ - URL: item.Url, + URL: cleanURL, Header: item.Title, - Description: item.Snippet, + Description: item.Description, + Source: "Qwant", }) } return results, nil } + +// cleanQwantURL extracts the main part of the URL, removing tracking information +func cleanQwantURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + return u.Scheme + "://" + u.Host + u.Path +} diff --git a/text.go b/text.go index cba25bb..b3a8e41 100644 --- a/text.go +++ b/text.go @@ -1,12 +1,13 @@ -// text.go package main import ( + "flag" "fmt" "html/template" "log" "net/http" "sort" + "sync" "time" ) @@ -14,49 +15,101 @@ type TextSearchResult struct { URL string Header string Description string + Source string } -func handleTextSearch(w http.ResponseWriter, query, safe, lang string) { - googleResults, googleErr := PerformGoogleTextSearch(query, safe, lang) - if googleErr != nil { - log.Printf("Error performing Google text search: %v", googleErr) +var debugMode bool + +func init() { + flag.BoolVar(&debugMode, "debug", false, "enable debug mode") + flag.Parse() +} + +func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) { + startTime := time.Now() + var combinedResults []TextSearchResult + var resultMap = make(map[string]TextSearchResult) + var wg sync.WaitGroup + var mu sync.Mutex + + resultsChan := make(chan []TextSearchResult) + + searchFuncs := []struct { + Func func(string, string, string) ([]TextSearchResult, error) + Source string + }{ + {PerformGoogleTextSearch, "Google"}, + {PerformDuckDuckGoTextSearch, "DuckDuckGo"}, + {PerformQwantTextSearch, "Qwant"}, } - qwantResults, qwantErr := PerformQwantTextSearch(query, safe, lang) - if qwantErr != nil { - log.Printf("Error performing Qwant text search: %v", qwantErr) + wg.Add(len(searchFuncs)) + + for _, searchFunc := range searchFuncs { + go func(searchFunc func(string, string, string) ([]TextSearchResult, error), source string) { + defer wg.Done() + results, err := searchFunc(query, safe, lang) + if err == nil { + for i := range results { + results[i].Source = source + } + resultsChan <- results + } else { + log.Printf("Error performing search from %s: %v", source, err) + } + }(searchFunc.Func, searchFunc.Source) } - // Use a map to track URLs and prioritize Qwant results - resultMap := make(map[string]TextSearchResult) + go func() { + wg.Wait() + close(resultsChan) + }() - // Add Qwant results to the map first - for _, result := range qwantResults { - resultMap[result.URL] = result - } - - // Add Google results to the map if the URL is not already present - for _, result := range googleResults { - if _, exists := resultMap[result.URL]; !exists { - resultMap[result.URL] = result + for results := range resultsChan { + mu.Lock() + for _, result := range results { + existingResult, exists := resultMap[result.URL] + if !exists || shouldReplace(existingResult.Source, result.Source) { + resultMap[result.URL] = result + } + if debugMode { + log.Printf("Result from %s: %+v\n", result.Source, result) + } } + mu.Unlock() } // Convert the map back to a slice - var combinedResults []TextSearchResult for _, result := range resultMap { combinedResults = append(combinedResults, result) } - // Sort results (optional, based on some criteria) + // Custom sorting: Google first, DuckDuckGo second, Qwant third sort.SliceStable(combinedResults, func(i, j int) bool { - return combinedResults[i].Header < combinedResults[j].Header + return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source) }) - displayResults(w, combinedResults, query, lang) + displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds()) } -func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string) { +func shouldReplace(existingSource, newSource string) bool { + return sourceOrder(newSource) < sourceOrder(existingSource) +} + +func sourceOrder(source string) int { + switch source { + case "Qwant": + return 1 + case "DuckDuckGo": + return 2 + case "Google": + return 3 + default: + return 4 + } +} + +func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64) { tmpl, err := template.ParseFiles("templates/text.html") if err != nil { http.Error(w, "Internal Server Error", http.StatusInternalServerError) @@ -72,7 +125,7 @@ func displayResults(w http.ResponseWriter, results []TextSearchResult, query, la }{ Results: results, Query: query, - Fetched: fmt.Sprintf("%.2f seconds", time.Since(time.Now()).Seconds()), + Fetched: fmt.Sprintf("%.2f seconds", elapsed), LanguageOptions: languageOptions, CurrentLang: lang, }