Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Compare commits

base: Spitfire:84d790a56e195d0915279fb17fcb079cacad32ac
Branches Tags
Spitfire:main
Spitfire:work
Spitfire:v0.3.0
Spitfire:v0.2.0
Spitfire:v0.1.2
Spitfire:v0.1.1
Spitfire:v0.1.0
Spitfire:v0.0.1
..
compare: Spitfire:3ad4c37d97f43c8c4eed2102c0d01812d44ac53d
Branches Tags
Spitfire:main
Spitfire:work
Spitfire:v0.3.0
Spitfire:v0.2.0
Spitfire:v0.1.2
Spitfire:v0.1.1
Spitfire:v0.1.0
Spitfire:v0.0.1

No commits in common. "84d790a56e195d0915279fb17fcb079cacad32ac" and "3ad4c37d97f43c8c4eed2102c0d01812d44ac53d" have entirely different histories.
84d790a56e ... 3ad4c37d97

2 changed files with 57 additions and 103 deletions
Show stats Expand all files Collapse all files

71
main.go
View file

@ -77,7 +77,38 @@ func main() {
}
func handleSearch(w http.ResponseWriter, r *http.Request) {
query, safe, lang, searchType, page := parseSearchParams(r)
var query, safe, lang, searchType string
var page int
if r.Method == "GET" {
query = r.URL.Query().Get("q")
safe = r.URL.Query().Get("safe")
lang = r.URL.Query().Get("lang")
searchType = r.URL.Query().Get("t")
pageStr := r.URL.Query().Get("p")
var err error
page, err = strconv.Atoi(pageStr)
if err != nil || page < 1 {
if debugMode {
log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
}
page = 1 // Default to page 1 if no valid page is specified
}
} else if r.Method == "POST" {
query = r.FormValue("q")
safe = r.FormValue("safe")
lang = r.FormValue("lang")
searchType = r.FormValue("t")
pageStr := r.FormValue("p")
var err error
page, err = strconv.Atoi(pageStr)
if err != nil || page < 1 {
if debugMode {
log.Printf("Invalid page parameter: %v, defaulting to page 1", err)
}
page = 1 // Default to page 1 if no valid page is specified
}
}
if query == "" {
http.ServeFile(w, r, "templates/search.html")
@ -85,47 +116,15 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
}
switch searchType {
case "text":
HandleTextSearch(w, query, safe, lang, page)
case "image":
handleImageSearch(w, query, safe, lang, page)
case "video":
videoSearchEndpointHandler(w, r)
case "map":
handleMapSearch(w, query, safe)
case "text":
fallthrough
default:
HandleTextSearch(w, query, safe, lang, page)
http.ServeFile(w, r, "templates/search.html")
}
}
func parseSearchParams(r *http.Request) (query, safe, lang, searchType string, page int) {
if r.Method == "GET" {
query = r.URL.Query().Get("q")
safe = r.URL.Query().Get("safe")
lang = r.URL.Query().Get("lang")
searchType = r.URL.Query().Get("t")
pageStr := r.URL.Query().Get("p")
page = parsePageParameter(pageStr)
} else if r.Method == "POST" {
query = r.FormValue("q")
safe = r.FormValue("safe")
lang = r.FormValue("lang")
searchType = r.FormValue("t")
pageStr := r.FormValue("p")
page = parsePageParameter(pageStr)
}
if searchType == "" {
searchType = "text"
}
return
}
func parsePageParameter(pageStr string) int {
page, err := strconv.Atoi(pageStr)
if err != nil || page < 1 {
page = 1
}
return page
}

89
text.go
View file

@ -26,20 +26,9 @@ func HandleTextSearch(w http.ResponseWriter, query, safe, lang string, page int)
const resultsPerPage = 10
cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
combinedResults := getResultsFromCacheOrFetch(cacheKey, query, safe, lang, page, resultsPerPage)
hasPrevPage := page > 1
hasNextPage := len(combinedResults) == resultsPerPage
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
// Always check and cache the next page
go cacheNextPageIfNotCached(query, safe, lang, page+1, resultsPerPage)
}
func getResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
cacheChan := make(chan []TextSearchResult)
var combinedResults []TextSearchResult
var fromCache bool
go func() {
results, exists := resultsCache.Get(cacheKey)
@ -54,55 +43,30 @@ func getResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, pag
select {
case combinedResults = <-cacheChan:
if combinedResults == nil {
combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
if combinedResults != nil {
fromCache = true
} else {
combinedResults = fetchAndCacheResults(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, combinedResults)
}
case <-time.After(2 * time.Second):
log.Println("Cache check timeout")
combinedResults = fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
combinedResults = fetchAndCacheResults(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, combinedResults)
}
return combinedResults
// Only pre-fetch and cache results for the next page if we fetched new results
if !fromCache {
go func() {
nextPageResults := fetchAndCacheResults(query, safe, lang, page+1, resultsPerPage)
resultsCache.Set(CacheKey{Query: query, Page: page + 1, Safe: safe, Lang: lang}, nextPageResults)
}()
}
func cacheNextPageIfNotCached(query, safe, lang string, page, resultsPerPage int) {
cacheKey := CacheKey{Query: query, Page: page, Safe: safe, Lang: lang}
if _, exists := resultsCache.Get(cacheKey); !exists {
log.Printf("Next page %d not cached, caching now...", page)
nextPageResults := fetchResultsUntilFull(query, safe, lang, page, resultsPerPage)
resultsCache.Set(cacheKey, nextPageResults)
} else {
log.Printf("Next page %d already cached", page)
}
}
hasPrevPage := page > 1
hasNextPage := len(combinedResults) == resultsPerPage
func fetchResultsUntilFull(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
var combinedResults []TextSearchResult
currentPage := 1
resultsNeeded := page * resultsPerPage
for len(combinedResults) < resultsNeeded {
results := fetchAndCacheResults(query, safe, lang, currentPage, resultsPerPage)
if len(results) == 0 {
break
}
combinedResults = append(combinedResults, results...)
currentPage++
}
startIndex := (page - 1) * resultsPerPage
endIndex := startIndex + resultsPerPage
if startIndex >= len(combinedResults) {
return []TextSearchResult{}
}
if endIndex > len(combinedResults) {
endIndex = len(combinedResults)
}
return combinedResults[startIndex:endIndex]
displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
}
func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []TextSearchResult {
@ -149,29 +113,24 @@ func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) []
mu.Unlock()
}
// Sort combinedResults by source priority: Google first, DuckDuckGo second, Qwant third
sort.SliceStable(combinedResults, func(i, j int) bool {
return sourceOrder(combinedResults[i].Source) < sourceOrder(combinedResults[j].Source)
})
log.Printf("Fetched %d results for page %d", len(combinedResults), page)
return combinedResults
}
func paginateResults(results []TextSearchResult, page, resultsPerPage int) []TextSearchResult {
// Paginate results
startIndex := (page - 1) * resultsPerPage
endIndex := startIndex + resultsPerPage
log.Printf("Paginating results: startIndex=%d, endIndex=%d, totalResults=%d", startIndex, endIndex, len(results))
if startIndex >= len(results) {
// Ensure startIndex and endIndex are within bounds
if startIndex >= len(combinedResults) {
return []TextSearchResult{}
}
if endIndex > len(results) {
endIndex = len(results)
if endIndex > len(combinedResults) {
endIndex = len(combinedResults)
}
return results[startIndex:endIndex]
return combinedResults[startIndex:endIndex]
}
func sourceOrder(source string) int {
@ -188,10 +147,6 @@ func sourceOrder(source string) int {
}
func displayResults(w http.ResponseWriter, results []TextSearchResult, query, lang string, elapsed float64, page int, hasPrevPage, hasNextPage bool) {
log.Printf("Displaying results for page %d", page)
log.Printf("Total results: %d", len(results))
log.Printf("Has previous page: %t, Has next page: %t", hasPrevPage, hasNextPage)
tmpl, err := template.New("text.html").Funcs(template.FuncMap{
"sub": func(a, b int) int {
return a - b