From a5622b9099c6baaa29d0b16e5c343c1cf8001b45 Mon Sep 17 00:00:00 2001 From: partisan Date: Wed, 25 Sep 2024 14:07:12 +0200 Subject: [PATCH] improved search-suggestions response time --- suggestions.go | 105 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 19 deletions(-) diff --git a/suggestions.go b/suggestions.go index 73e9b02..d11f2be 100644 --- a/suggestions.go +++ b/suggestions.go @@ -6,8 +6,62 @@ import ( "io" "net/http" "net/url" + "sort" + "sync" + "time" ) +// SuggestionSource represents a search suggestion source along with its latency. +type SuggestionSource struct { + Name string + FetchFunc func(string) []string + Latency time.Duration + mu sync.Mutex +} + +// Initialize suggestion sources with default latency values. +var suggestionSources = []SuggestionSource{ + { + Name: "DuckDuckGo", + FetchFunc: fetchDuckDuckGoSuggestions, + Latency: 50 * time.Millisecond, + }, + { + Name: "Edge", + FetchFunc: fetchEdgeSuggestions, + Latency: 50 * time.Millisecond, + }, + { + Name: "Brave", + FetchFunc: fetchBraveSuggestions, + Latency: 50 * time.Millisecond, + }, + { + Name: "Ecosia", + FetchFunc: fetchEcosiaSuggestions, + Latency: 50 * time.Millisecond, + }, + { + Name: "Qwant", + FetchFunc: fetchQwantSuggestions, + Latency: 50 * time.Millisecond, + }, + { + Name: "Startpage", + FetchFunc: fetchStartpageSuggestions, + Latency: 50 * time.Millisecond, + }, + // I advise against it, but you can use it if you want to + // { + // Name: "Google", + // FetchFunc: fetchGoogleSuggestions, + // Latency: 500 * time.Millisecond, + // }, +} + +// Mutex to protect the suggestionSources during sorting. +var suggestionsMU sync.Mutex + func handleSuggestions(w http.ResponseWriter, r *http.Request) { query := r.URL.Query().Get("q") if query == "" { @@ -16,25 +70,27 @@ func handleSuggestions(w http.ResponseWriter, r *http.Request) { return } - // Define the fallback sequence with Google lower in the hierarchy - suggestionSources := []func(string) []string{ - fetchDuckDuckGoSuggestions, - fetchEdgeSuggestions, - fetchBraveSuggestions, - fetchEcosiaSuggestions, - fetchQwantSuggestions, - fetchStartpageSuggestions, - // fetchGoogleSuggestions, // I advise against it, but you can use it if you want to - } + // Sort the suggestion sources based on their latency. + suggestionsMU.Lock() + sort.Slice(suggestionSources, func(i, j int) bool { + return suggestionSources[i].Latency < suggestionSources[j].Latency + }) + suggestionsMU.Unlock() var suggestions []string - for _, fetchFunc := range suggestionSources { - suggestions = fetchFunc(query) + for i := range suggestionSources { + source := &suggestionSources[i] + start := time.Now() + suggestions = source.FetchFunc(query) + elapsed := time.Since(start) + + updateLatency(source, elapsed) + if len(suggestions) > 0 { - printDebug("Suggestions found using %T", fetchFunc) + printDebug("Suggestions found using %s", source.Name) break } else { - printWarn("%T did not return any suggestions or failed.", fetchFunc) + printWarn("%s did not return any suggestions or failed.", source.Name) } } @@ -42,11 +98,19 @@ func handleSuggestions(w http.ResponseWriter, r *http.Request) { printErr("All suggestion services failed. Returning empty response.") } - // Return the final suggestions as JSON + // Return the final suggestions as JSON. w.Header().Set("Content-Type", "application/json") fmt.Fprintf(w, `["",%s]`, toJSONStringArray(suggestions)) } +// updateLatency updates the latency of a suggestion source using an exponential moving average. +func updateLatency(source *SuggestionSource, newLatency time.Duration) { + source.mu.Lock() + defer source.mu.Unlock() + const alpha = 0.5 // Smoothing factor. + source.Latency = time.Duration(float64(source.Latency)*(1-alpha) + float64(newLatency)*alpha) +} + func fetchGoogleSuggestions(query string) []string { encodedQuery := url.QueryEscape(query) url := fmt.Sprintf("http://suggestqueries.google.com/complete/search?client=firefox&q=%s", encodedQuery) @@ -82,6 +146,7 @@ func fetchEcosiaSuggestions(query string) []string { return fetchSuggestionsFromURL(url) } +// Is this working? func fetchQwantSuggestions(query string) []string { encodedQuery := url.QueryEscape(query) url := fmt.Sprintf("https://api.qwant.com/v3/suggest?q=%s", encodedQuery) @@ -96,6 +161,7 @@ func fetchStartpageSuggestions(query string) []string { return fetchSuggestionsFromURL(url) } +// fetchSuggestionsFromURL fetches suggestions from the given URL. func fetchSuggestionsFromURL(url string) []string { resp, err := http.Get(url) if err != nil { @@ -110,17 +176,17 @@ func fetchSuggestionsFromURL(url string) []string { return []string{} } - // Log the Content-Type for debugging + // Log the Content-Type for debugging. contentType := resp.Header.Get("Content-Type") printDebug("Response Content-Type from %s: %s", url, contentType) - // Check if the body is non-empty + // Check if the body is non-empty. if len(body) == 0 { printWarn("Received empty response body from %s", url) return []string{} } - // Attempt to parse the response as JSON regardless of Content-Type + // Attempt to parse the response as JSON regardless of Content-Type. var parsedResponse []interface{} if err := json.Unmarshal(body, &parsedResponse); err != nil { printErr("Error parsing JSON from %s: %v", url, err) @@ -128,7 +194,7 @@ func fetchSuggestionsFromURL(url string) []string { return []string{} } - // Ensure the response structure is as expected + // Ensure the response structure is as expected. if len(parsedResponse) < 2 { printWarn("Unexpected response format from %v: %v", url, string(body)) return []string{} @@ -148,6 +214,7 @@ func fetchSuggestionsFromURL(url string) []string { return suggestions } +// toJSONStringArray converts a slice of strings to a JSON array string. func toJSONStringArray(strings []string) string { result := "" for i, str := range strings {