Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

cache v1 + debug mode + buttons on text results

Browse source
This commit is contained in:
partisan 2024-05-19 22:57:23 +02:00
parent e78667ab85
commit 18fe1baf0d
9 changed files with 228 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
cache.go Normal file
View file

@ -0,0 +1,55 @@
package main
import (
"fmt"
"sync"
)
// TextSearchResult represents a single search result item.
type TextSearchResult struct {
URL string
Header string
Description string
Source string
}
// CacheKey represents the key used to store search results in the cache.
type CacheKey struct {
Query string
Page int
Safe string
Lang string
}
// ResultsCache is a thread-safe map for caching search results by composite keys.
type ResultsCache struct {
mu sync.Mutex
results map[string][]TextSearchResult
}
// NewResultsCache creates a new ResultsCache.
func NewResultsCache() *ResultsCache {
return &ResultsCache{
results: make(map[string][]TextSearchResult),
}
}
// Get retrieves the results for a given key from the cache.
func (rc *ResultsCache) Get(key CacheKey) ([]TextSearchResult, bool) {
rc.mu.Lock()
defer rc.mu.Unlock()
results, exists := rc.results[rc.keyToString(key)]
return results, exists
}
// Set stores the results for a given key in the cache.
func (rc *ResultsCache) Set(key CacheKey, results []TextSearchResult) {
rc.mu.Lock()
defer rc.mu.Unlock()
rc.results[rc.keyToString(key)] = results
}
// keyToString converts a CacheKey to a string representation.
func (rc *ResultsCache) keyToString(key CacheKey) string {
return fmt.Sprintf("%s|%d|%s|%s", key.Query, key.Page, key.Safe, key.Lang)
}

7
images.go
View file

@ -56,9 +56,9 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch
offset = (page - 1) * resultsPerPage
}
// Ensuring safe search is enabled by default if not specified
// Ensuring safe search is disabled by default if not specified
if safe == "" {
safe = "1"
safe = "0"
}
// Defaulting to English Canada locale if not specified
@ -66,8 +66,7 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch
lang = "en_CA"
}
// Format &lang=lang_de is incorret, implement fix !
// Format &lang=lang_de is incorrect, implement fix !
apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/images?t=images&q=%s&count=%d&locale=%s&offset=%d&device=desktop&tgp=2&safesearch=%s",
url.QueryEscape(query),
resultsPerPage,

15
main.go
View file

@ -1,3 +1,4 @@
// main.go
package main
import (
@ -88,6 +89,9 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
var err error
page, err = strconv.Atoi(pageStr)
if err != nil || page < 1 {
if debugMode {
log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
}
page = 1 // Default to page 1 if no valid page is specified
}
} else if r.Method == "POST" {
@ -95,6 +99,15 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
safe = r.FormValue("safe")
lang = r.FormValue("lang")
searchType = r.FormValue("t")
pageStr := r.FormValue("p")
var err error
page, err = strconv.Atoi(pageStr)
if err != nil || page < 1 {
if debugMode {
log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
}
page = 1 // Default to page 1 if no valid page is specified
}
}
if query == "" {
@ -104,7 +117,7 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
switch searchType {
case "text":
HandleTextSearch(w, query, safe, lang)
HandleTextSearch(w, query, safe, lang, page)
case "image":
handleImageSearch(w, query, safe, lang, page)
case "video":

2
run.sh
View file

@ -1,3 +1,3 @@
#!/bin/bash
go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go --debug
go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go --debug

15
templates/text.html
View file

@ -42,7 +42,6 @@
<button name="t" value="torrent" class="clickable">Torrents</button>
</div>
</div>
</div>
</form>
<form class="results_settings" action="/search" method="get">
<input type="hidden" name="q" value="{{ .Query }}">
@ -58,7 +57,6 @@
<button class="results-save" name="t" value="text">Apply settings</button>
</form>
<div class="results">
<!-- Results go here -->
{{if .Results}}
{{range .Results}}
<div class="result_item">
@ -72,7 +70,18 @@
<div class="no-results">No results found for '{{ .Query }}'. Try different keywords.</div>
{{end}}
</div>
<div class="prev-next prev-img">
<form action="/search" method="get">
<input type="hidden" name="q" value="{{ .Query }}">
<input type="hidden" name="t" value="text">
{{ if .HasPrevPage }}
<button type="submit" name="p" value="{{ sub .Page 1 }}">Previous</button>
{{ end }}
{{ if .HasNextPage }}
<button type="submit" name="p" value="{{ add .Page 1 }}">Next</button>
{{ end }}
</form>
</div>
<script>
// Check if JavaScript is enabled and modify the DOM accordingly
document.getElementById('content').classList.remove('js-enabled');

35
text-duckduckgo.go
View file

@ -7,15 +7,26 @@ import (
"net/http"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
func PerformDuckDuckGoTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
const resultsPerPage = 10
var results []TextSearchResult
searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s", url.QueryEscape(query))
resp, err := http.Get(searchURL)
client := &http.Client{Timeout: 10 * time.Second}
searchURL := fmt.Sprintf("https://duckduckgo.com/html/?q=%s&s=%d", url.QueryEscape(query), (page-1)*resultsPerPage)
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("making request: %v", err)
}
@ -49,10 +60,28 @@ func PerformDuckDuckGoTextSearch(query, safe, lang string) ([]TextSearchResult,
if debugMode {
log.Printf("Processed DuckDuckGo result: %+v\n", result)
}
} else {
if debugMode {
log.Printf("Missing 'uddg' parameter in URL: %s\n", rawURL)
}
}
} else {
if debugMode {
log.Printf("Error parsing URL: %s, error: %v\n", rawURL, err)
}
}
} else {
if debugMode {
log.Printf("Missing 'href' attribute in result anchor tag\n")
}
}
})
if len(results) == 0 {
if debugMode {
log.Println("No results found from DuckDuckGo")
}
}
return results, nil
}

35
text-google.go
View file

@ -2,15 +2,18 @@
package main
import (
"fmt"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
)
func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
const resultsPerPage = 10
var results []TextSearchResult
client := &http.Client{}
@ -24,29 +27,43 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
langParam = "&lr=" + lang
}
searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14"
// Calculate the start index based on the page number
startIndex := (page - 1) * resultsPerPage
searchURL := "https://www.google.com/search?q=" + url.QueryEscape(query) + safeParam + langParam + "&udm=14&start=" + strconv.Itoa(startIndex)
req, err := http.NewRequest("GET", searchURL, nil)
if err != nil {
log.Fatalf("Failed to create request: %v", err)
return nil, fmt.Errorf("failed to create request: %v", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return nil, err
return nil, fmt.Errorf("making request: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return nil, err
return nil, fmt.Errorf("loading HTML document: %v", err)
}
doc.Find(".yuRUbf").Each(func(i int, s *goquery.Selection) {
link := s.Find("a")
href, _ := link.Attr("href")
href, exists := link.Attr("href")
if !exists {
if debugMode {
log.Printf("No href attribute found for result %d\n", i)
}
return
}
header := link.Find("h3").Text()
header = strings.TrimSpace(strings.TrimSuffix(header, "›"))
@ -67,5 +84,11 @@ func PerformGoogleTextSearch(query, safe, lang string) ([]TextSearchResult, erro
}
})
if len(results) == 0 {
if debugMode {
log.Println("No results found from Google")
}
}
return results, nil
}

16
text-quant.go
View file

@ -1,8 +1,10 @@
// text-qwant.go
package main
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"time"
@ -26,9 +28,11 @@ type QwantTextAPIResponse struct {
}
// PerformQwantTextSearch contacts the Qwant API and returns a slice of TextSearchResult
func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error) {
func PerformQwantTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
const resultsPerPage = 10
const offset = 0
// Calculate the offset based on the page number
offset := (page - 1) * resultsPerPage
// Ensure safe search is disabled by default if not specified
if safe == "" {
@ -40,11 +44,12 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
lang = "en_CA"
}
apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop",
apiURL := fmt.Sprintf("https://api.qwant.com/v3/search/web?q=%s&count=%d&locale=%s&offset=%d&device=desktop&safesearch=%s",
url.QueryEscape(query),
resultsPerPage,
lang,
offset)
offset,
safe)
client := &http.Client{Timeout: 10 * time.Second}
@ -93,6 +98,9 @@ func PerformQwantTextSearch(query, safe, lang string) ([]TextSearchResult, error
func cleanQwantURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
if debugMode {
log.Printf("Error parsing URL: %v", err)
}
return rawURL
}
return u.Scheme + "://" + u.Host + u.Path

101
text.go
View file

@ -1,3 +1,4 @@
// text.go
package main
import (
@ -11,31 +12,49 @@ import (
"time"
)
type TextSearchResult struct {
URL string
Header string
Description string
Source string
}
var debugMode bool
var (
debugMode bool
resultsCache = NewResultsCache()
)
func init() {
flag.BoolVar(&debugMode, "debug", false, "enable debug mode")
flag.Parse()
}
func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int) {
startTime := time.Now()
const resultsPerPage = 10
cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
// Try to get results from cache
combinedResults, exists := resultsCache.Get(cacheKey)
if !exists {
// Fetch results for the current page
combinedResults = fetchAndCacheResults(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, combinedResults)
}
// Pre-fetch and cache results for the next page
nextPageResults := fetchAndCacheResults(query, safe, lang, page+1, resultsPerPage)
resultsCache.Set(CacheKey{Query: query, Page: page + 1, Safe: safe, Lang: lang}, nextPageResults)
hasPrevPage := page > 1
hasNextPage := len(nextPageResults) > 0
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
}
func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
var resultMap = make(map[string]TextSearchResult)
var wg sync.WaitGroup
var mu sync.Mutex
resultsChan := make(chan []TextSearchResult)
searchFuncs := []struct {
Func func(string, string, string) ([]TextSearchResult, error)
Func func(string, string, string, int) ([]TextSearchResult, error)
Source string
}{
{PerformGoogleTextSearch, "Google"},
@ -46,9 +65,9 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
wg.Add(len(searchFuncs))
for _, searchFunc := range searchFuncs {
go func(searchFunc func(string, string, string) ([]TextSearchResult, error), source string) {
go func(searchFunc func(string, string, string, int) ([]TextSearchResult, error), source string) {
defer wg.Done()
results, err := searchFunc(query, safe, lang)
results, err := searchFunc(query, safe, lang, page)
if err == nil {
for i := range results {
results[i].Source = source
@ -67,50 +86,52 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string) {
for results := range resultsChan {
mu.Lock()
for _, result := range results {
existingResult, exists := resultMap[result.URL]
if !exists || shouldReplace(existingResult.Source, result.Source) {
resultMap[result.URL] = result
}
if debugMode {
log.Printf("Result from %s: %+v\n", result.Source, result)
}
}
combinedResults = append(combinedResults, results...)
mu.Unlock()
}
// Convert the map back to a slice
for _, result := range resultMap {
combinedResults = append(combinedResults, result)
}
// Custom sorting: Google first, DuckDuckGo second, Qwant third
// Sort combinedResults by source priority: Google first, DuckDuckGo second, Qwant third
sort.SliceStable(combinedResults, func(i, j int) bool {
return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
})
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds())
// Paginate results
startIndex := (page - 1) * resultsPerPage
endIndex := startIndex + resultsPerPage
// Ensure startIndex and endIndex are within bounds
if startIndex >= len(combinedResults) {
return []TextSearchResult{}
}
if endIndex > len(combinedResults) {
endIndex = len(combinedResults)
}
func shouldReplace(existingSource, newSource string) bool {
return sourceOrder(newSource) < sourceOrder(existingSource)
return combinedResults[startIndex:endIndex]
}
func sourceOrder(source string) int {
switch source {
case "Qwant":
return 3
case "DuckDuckGo":
return 2
case "Google":
return 1
case "DuckDuckGo":
return 2
case "Qwant":
return 3
default:
return 4
}
}
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64) {
tmpl, err := template.ParseFiles("templates/text.html")
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64, page int, hasPrevPage, hasNextPage bool) {
tmpl, err := template.New("text.html").Funcs(template.FuncMap{
"sub": func(a, b int) int {
return a - b
},
"add": func(a, b int) int {
return a + b
},
}).ParseFiles("templates/text.html")
if err != nil {
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
@ -120,12 +141,18 @@ func displayResults(w http.ResponseWriter, results []TextSearchResult, query, la
Results []TextSearchResult
Query string
Fetched string
Page int
HasPrevPage bool
HasNextPage bool
LanguageOptions []LanguageOption
CurrentLang string
}{
Results: results,
Query: query,
Fetched: fmt.Sprintf("%.2f seconds", elapsed),
Page: page,
HasPrevPage: hasPrevPage,
HasNextPage: hasNextPage,
LanguageOptions: languageOptions,
CurrentLang: lang,
}