Spitfire/Search
Spitfire/Search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

cleanup

Browse source
This commit is contained in:
partisan 2024-08-13 16:31:28 +02:00
parent 04fb0ca629
commit e056832321
16 changed files with 2275 additions and 2252 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
agent.go Normal file → Executable file
View file

@ -260,7 +260,7 @@ func periodicUpdate() {
// Fetch the latest browser versions
newVersions, err := fetchLatestBrowserVersions()
if err != nil {
fmt.Println("Error fetching latest browser versions:", err)
printWarn("Error fetching latest browser versions: %v", err)
continue
}

3
common.go Normal file → Executable file
View file

@ -4,7 +4,6 @@ import (
"crypto/rand"
"encoding/base64"
"html/template"
"log"
"strings"
)
@ -23,7 +22,7 @@ func generateStrongRandomString(length int) string {
bytes := make([]byte, length)
_, err := rand.Read(bytes)
if err != nil {
log.Fatalf("Error generating random string: %v", err)
printErr("Error generating random string: %v", err)
}
return base64.URLEncoding.EncodeToString(bytes)[:length]
}

27
files.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"fmt"
"html/template"
"log"
"net/http"
"net/url"
"regexp"
@ -39,11 +38,11 @@ func initializeTorrentSites() {
// rutor = NewRutor()
}
func handleFileSearch(w http.ResponseWriter, settings UserSettings, query, safe, lang string, page int) {
func handleFileSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
startTime := time.Now()
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "file"}
combinedResults := getFileResultsFromCacheOrFetch(cacheKey, query, safe, lang, page)
cacheKey := CacheKey{Query: query, Page: page, Safe: settings.SafeSearch == "true", Lang: settings.Language, Type: "file"}
combinedResults := getFileResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.Language, page)
sort.Slice(combinedResults, func(i, j int) bool { return combinedResults[i].Seeders > combinedResults[j].Seeders })
@ -54,7 +53,7 @@ func handleFileSearch(w http.ResponseWriter, settings UserSettings, query, safe,
}
tmpl, err := template.New("files.html").Funcs(funcMap).ParseFiles("templates/files.html")
if err != nil {
log.Printf("Failed to load template: %v", err)
printErr("Failed to load template: %v", err)
http.Error(w, "Failed to load template", http.StatusInternalServerError)
return
}
@ -79,7 +78,7 @@ func handleFileSearch(w http.ResponseWriter, settings UserSettings, query, safe,
HasPrevPage: page > 1,
HasNextPage: len(combinedResults) > 0,
Page: page,
Settings: Settings{UxLang: lang, Safe: safe}, // Now this is painful, are there two Settings variables??
Settings: Settings{UxLang: settings.Language, Safe: settings.SafeSearch}, // Now this is painful, are there two Settings variables??
Theme: settings.Theme,
}
@ -89,7 +88,7 @@ func handleFileSearch(w http.ResponseWriter, settings UserSettings, query, safe,
// }
if err := tmpl.Execute(w, data); err != nil {
log.Printf("Failed to render template: %v", err)
printErr("Failed to render template: %v", err)
http.Error(w, "Failed to render template", http.StatusInternalServerError)
}
}
@ -101,10 +100,10 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
go func() {
results, exists := resultsCache.Get(cacheKey)
if exists {
log.Println("Cache hit")
printInfo("Cache hit")
cacheChan <- results
} else {
log.Println("Cache miss")
printInfo("Cache miss")
cacheChan <- nil
}
}()
@ -121,7 +120,7 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
combinedResults = torrentResults
}
case <-time.After(2 * time.Second):
log.Println("Cache check timeout")
printInfo("Cache check timeout")
combinedResults = fetchFileResults(query, safe, lang, page)
if len(combinedResults) > 0 {
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@ -150,7 +149,7 @@ func fetchFileResults(query, safe, lang string, page int) []TorrentResult {
}
if len(results) == 0 {
log.Printf("No file results found for query: %s, trying other nodes", query)
printWarn("No file results found for query: %s, trying other nodes", query)
results = tryOtherNodesForFileSearch(query, safe, lang, page, []string{hostID})
}
@ -180,7 +179,7 @@ func parseSize(sizeStr string) int64 {
re := regexp.MustCompile(`(?i)([\d.]+)\s*([KMGT]?B)`)
matches := re.FindStringSubmatch(sizeStr)
if len(matches) < 3 {
log.Printf("Error parsing size: invalid format %s", sizeStr)
printWarn("Error parsing size: invalid format %s", sizeStr)
return 0
}
@ -198,13 +197,13 @@ func parseSize(sizeStr string) int64 {
case "TB":
multiplier = 1024 * 1024 * 1024 * 1024
default:
log.Printf("Unknown unit: %s", unit)
printWarn("Unknown unit: %s", unit)
return 0
}
size, err := strconv.ParseFloat(sizeStr, 64)
if err != nil {
log.Printf("Error parsing size: %v", err)
printWarn("Error parsing size: %v", err)
return 0
}
return int64(size * float64(multiplier))

8
forums.go Normal file → Executable file
View file

@ -98,11 +98,11 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu
return results, nil
}
func handleForumsSearch(w http.ResponseWriter, settings UserSettings, query, safe, lang string, page int) {
results, err := PerformRedditSearch(query, safe, page)
func handleForumsSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
results, err := PerformRedditSearch(query, settings.SafeSearch, page)
if err != nil || len(results) == 0 { // 0 == 0 to force search by other node
log.Printf("No results from primary search, trying other nodes")
results = tryOtherNodesForForumSearch(query, safe, lang, page)
results = tryOtherNodesForForumSearch(query, settings.SafeSearch, settings.Language, page)
}
data := struct {
@ -118,7 +118,7 @@ func handleForumsSearch(w http.ResponseWriter, settings UserSettings, query, saf
Query: query,
Results: results,
LanguageOptions: languageOptions,
CurrentLang: lang,
CurrentLang: settings.Language,
Page: page,
HasPrevPage: page > 1,
HasNextPage: len(results) == 25,

22
images.go Normal file → Executable file
View file

@ -18,11 +18,11 @@ func init() {
}
}
func handleImageSearch(w http.ResponseWriter, settings UserSettings, query, safe, lang string, page int) {
func handleImageSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
startTime := time.Now()
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "image"}
combinedResults := getImageResultsFromCacheOrFetch(cacheKey, query, safe, lang, page)
cacheKey := CacheKey{Query: query, Page: page, Safe: settings.SafeSearch == "true", Lang: settings.Language, Type: "image"}
combinedResults := getImageResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.Language, page)
elapsedTime := time.Since(startTime)
tmpl, err := template.New("images.html").Funcs(funcs).ParseFiles("templates/images.html")
@ -49,7 +49,7 @@ func handleImageSearch(w http.ResponseWriter, settings UserSettings, query, safe
Page: page,
Fetched: fmt.Sprintf("%.2f seconds", elapsedTime.Seconds()),
LanguageOptions: languageOptions,
CurrentLang: lang,
CurrentLang: settings.Language,
HasPrevPage: page > 1,
HasNextPage: len(combinedResults) >= 50,
NoResults: len(combinedResults) == 0,
@ -58,7 +58,7 @@ func handleImageSearch(w http.ResponseWriter, settings UserSettings, query, safe
err = tmpl.Execute(w, data)
if err != nil {
log.Printf("Error executing template: %v", err)
printErr("Error executing template: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
}
@ -70,10 +70,10 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
go func() {
results, exists := resultsCache.Get(cacheKey)
if exists {
log.Println("Cache hit")
printInfo("Cache hit")
cacheChan <- results
} else {
log.Println("Cache miss")
printInfo("Cache miss")
cacheChan <- nil
}
}()
@ -90,7 +90,7 @@ func getImageResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string
combinedResults = imageResults
}
case <-time.After(2 * time.Second):
log.Println("Cache check timeout")
printInfo("Cache check timeout")
combinedResults = fetchImageResults(query, safe, lang, page)
if len(combinedResults) > 0 {
resultsCache.Set(cacheKey, convertToSearchResults(combinedResults))
@ -104,12 +104,12 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
var results []ImageSearchResult
for _, engine := range imageSearchEngines {
log.Printf("Using image search engine: %s", engine.Name)
printInfo("Using image search engine: %s", engine.Name)
searchResults, duration, err := engine.Func(query, safe, lang, page)
updateEngineMetrics(&engine, duration, err == nil)
if err != nil {
log.Printf("Error performing image search with %s: %v", engine.Name, err)
printWarn("Error performing image search with %s: %v", engine.Name, err)
continue
}
@ -125,7 +125,7 @@ func fetchImageResults(query, safe, lang string, page int) []ImageSearchResult {
// If no results found after trying all engines
if len(results) == 0 {
log.Printf("No image results found for query: %s, trying other nodes", query)
printWarn("No image results found for query: %s, trying other nodes", query)
results = tryOtherNodesForImageSearch(query, safe, lang, page, []string{hostID})
}

13
main.go Normal file → Executable file
View file

@ -103,21 +103,20 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
// Handle search based on the type
switch searchType {
case "image":
handleImageSearch(w, settings, query, safe, lang, page)
handleImageSearch(w, settings, query, page)
case "video":
handleVideoSearch(w, settings, query, safe, lang, page)
handleVideoSearch(w, settings, query, page)
case "map":
handleMapSearch(w, settings, query, safe)
handleMapSearch(w, settings, query)
case "forum":
handleForumsSearch(w, settings, query, safe, lang, page)
handleForumsSearch(w, settings, query, page)
case "file":
handleFileSearch(w, settings, query, safe, lang, page)
handleFileSearch(w, settings, query, page)
case "text":
fallthrough
default:
HandleTextSearch(w, settings, query, safe, lang, page)
HandleTextSearch(w, settings, query, page)
}
// This is immeasurably stupid it passes safe and language then it passes settings with safe and lang again
}
func parseSearchParams(r *http.Request) (query, safe, lang, searchType string, page int) {

2
map.go Normal file → Executable file
View file

@ -43,7 +43,7 @@ func geocodeQuery(query string) (latitude, longitude string, found bool, err err
return "", "", false, nil
}
func handleMapSearch(w http.ResponseWriter, settings UserSettings, query string, lang string) {
func handleMapSearch(w http.ResponseWriter, settings UserSettings, query string) {
// Geocode the query to get coordinates
latitude, longitude, found, err := geocodeQuery(query)
if err != nil {

12
node-handle-search.go Normal file → Executable file
View file

@ -15,16 +15,16 @@ func handleSearchTextMessage(msg Message) {
}
err := json.Unmarshal([]byte(msg.Content), &searchParams)
if err != nil {
log.Printf("Error parsing search parameters: %v", err)
printWarn("Error parsing search parameters: %v", err)
return
}
log.Printf("Received search-text request. ResponseAddr: %s", searchParams.ResponseAddr)
printDebug("Received search-text request. ResponseAddr: %s", searchParams.ResponseAddr)
results := fetchTextResults(searchParams.Query, searchParams.Safe, searchParams.Lang, searchParams.Page)
resultsJSON, err := json.Marshal(results)
if err != nil {
log.Printf("Error marshalling search results: %v", err)
printWarn("Error marshalling search results: %v", err)
return
}
@ -35,16 +35,16 @@ func handleSearchTextMessage(msg Message) {
}
// Log the address to be used for sending the response
log.Printf("Sending text search results to %s", searchParams.ResponseAddr)
printDebug("Sending text search results to %s", searchParams.ResponseAddr)
if searchParams.ResponseAddr == "" {
log.Printf("Error: Response address is empty")
printErr("Error: Response address is empty")
return
}
err = sendMessage(searchParams.ResponseAddr, responseMsg)
if err != nil {
log.Printf("Error sending text search results to %s: %v", searchParams.ResponseAddr, err)
printWarn("Error sending text search results to %s: %v", searchParams.ResponseAddr, err)
}
}

7
node-request-files.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"log"
"time"
)
@ -14,7 +13,7 @@ func tryOtherNodesForFileSearch(query, safe, lang string, page int, visitedNodes
}
results, err := sendFileSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
printWarn("Error contacting node %s: %v", nodeAddr, err)
continue
}
if len(results) > 0 {
@ -71,11 +70,11 @@ func handleFileResultsMessage(msg Message) {
var results []TorrentResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling file results: %v", err)
printWarn("Error unmarshalling file results: %v", err)
return
}
log.Printf("Received file results: %+v", results)
printDebug("Received file results: %+v", results)
// Send results to fileResultsChan
go func() {
fileResultsChan <- results

9
node-request-forums.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"log"
"time"
)
@ -13,7 +12,7 @@ func tryOtherNodesForForumSearch(query, safe, lang string, page int) []ForumSear
for _, nodeAddr := range peers {
results, err := sendForumSearchRequestToNode(nodeAddr, query, safe, lang, page, []string{})
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
printWarn("Error contacting node %s: %v", nodeAddr, err)
continue
}
if len(results) > 0 {
@ -79,11 +78,11 @@ func handleForumResultsMessage(msg Message) {
var results []ForumSearchResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling forum results: %v", err)
printWarn("Error unmarshalling forum results: %v", err)
return
}
log.Printf("Received forum results: %+v", results)
printDebug("Received forum results: %+v", results)
// Send results to forumResultsChan
go func() {
forumResultsChan <- results
@ -94,7 +93,7 @@ func handleForumResultsMessage(msg Message) {
func fetchForumResults(query, safe, lang string, page int) []ForumSearchResult {
results, err := PerformRedditSearch(query, safe, page)
if err != nil {
log.Printf("Error fetching forum results: %v", err)
printWarn("Error fetching forum results: %v", err)
return nil
}
return results

7
node-request-images.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"log"
"time"
)
@ -13,11 +12,11 @@ func handleImageResultsMessage(msg Message) {
var results []ImageSearchResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling image results: %v", err)
printWarn("Error unmarshalling image results: %v", err)
return
}
log.Printf("Received image results: %+v", results)
printDebug("Received image results: %+v", results)
// Send results to imageResultsChan
go func() {
imageResultsChan <- results
@ -74,7 +73,7 @@ func tryOtherNodesForImageSearch(query, safe, lang string, page int, visitedNode
}
results, err := sendImageSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
printWarn("Error contacting node %s: %v", nodeAddr, err)
continue
}
if len(results) > 0 {

7
node-request-text.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"log"
"time"
)
@ -16,7 +15,7 @@ func tryOtherNodesForTextSearch(query, safe, lang string, page int, visitedNodes
}
results, err := sendTextSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
printWarn("Error contacting node %s: %v", nodeAddr, err)
continue
}
if len(results) > 0 {
@ -73,11 +72,11 @@ func handleTextResultsMessage(msg Message) {
var results []TextSearchResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling text results: %v", err)
printWarn("Error unmarshalling text results: %v", err)
return
}
log.Printf("Received text results: %+v", results)
printDebug("Received text results: %+v", results)
// Send results to textResultsChan
go func() {
textResultsChan <- results

7
node-request-video.go Normal file → Executable file
View file

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"fmt"
"log"
"time"
)
@ -14,7 +13,7 @@ func tryOtherNodesForVideoSearch(query, safe, lang string, page int, visitedNode
}
results, err := sendVideoSearchRequestToNode(nodeAddr, query, safe, lang, page, visitedNodes)
if err != nil {
log.Printf("Error contacting node %s: %v", nodeAddr, err)
printWarn("Error contacting node %s: %v", nodeAddr, err)
continue
}
if len(results) > 0 {
@ -71,11 +70,11 @@ func handleVideoResultsMessage(msg Message) {
var results []VideoResult
err := json.Unmarshal([]byte(msg.Content), &results)
if err != nil {
log.Printf("Error unmarshalling video results: %v", err)
printWarn("Error unmarshalling video results: %v", err)
return
}
log.Printf("Received video results: %+v", results)
printDebug("Received video results: %+v", results)
// Send results to videoResultsChan
go func() {
videoResultsChan <- results

31
run.bat Executable file
View file

@ -0,0 +1,31 @@
@echo off
setlocal enabledelayedexpansion
rem Directory where the Go files are located
set GO_DIR=C:\path\to\your\go\files
rem Explicitly list the main files in the required order
set FILES=main.go init.go search-engine.go text.go text-google.go text-librex.go text-brave.go text-duckduckgo.go common.go cache.go agent.go files.go files-thepiratebay.go files-torrentgalaxy.go forums.go get-searchxng.go imageproxy.go images.go images-imgur.go images-quant.go map.go node.go open-search.go video.go
rem Change to the directory with the Go files
pushd %GO_DIR%
rem Find all other .go files that were not explicitly listed
set OTHER_GO_FILES=
for %%f in (*.go) do (
set file=%%~nxf
set found=0
for %%i in (%FILES%) do (
if /i "%%i"=="!file!" set found=1
)
if !found!==0 (
set OTHER_GO_FILES=!OTHER_GO_FILES! "%%f"
)
)
rem Run the Go program with the specified files first, followed by the remaining files
go run %FILES% %OTHER_GO_FILES%
rem Return to the original directory
popd

12
text.go Normal file → Executable file
View file

@ -19,20 +19,20 @@ func init() {
}
}
func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query, safe, lang string, page int) {
func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
startTime := time.Now()
cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "text"}
combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, safe, lang, page)
cacheKey := CacheKey{Query: query, Page: page, Safe: settings.SafeSearch == "true", Lang: settings.Language, Type: "text"}
combinedResults := getTextResultsFromCacheOrFetch(cacheKey, query, settings.SafeSearch, settings.Language, page)
hasPrevPage := page > 1 // dupe
//displayResults(w, combinedResults, query, lang, time.Since(startTime).Seconds(), page, hasPrevPage, hasNextPage)
// Prefetch next and previous pages
go prefetchPage(query, safe, lang, page+1)
go prefetchPage(query, settings.SafeSearch, settings.Language, page+1)
if hasPrevPage {
go prefetchPage(query, safe, lang, page-1)
go prefetchPage(query, settings.SafeSearch, settings.Language, page-1)
}
elapsedTime := time.Since(startTime)
@ -60,7 +60,7 @@ func HandleTextSearch(w http.ResponseWriter, settings UserSettings, query, safe,
Page: page,
Fetched: fmt.Sprintf("%.2f seconds", elapsedTime.Seconds()),
LanguageOptions: languageOptions,
CurrentLang: lang,
CurrentLang: settings.Language,
HasPrevPage: page > 1,
HasNextPage: len(combinedResults) >= 50,
NoResults: len(combinedResults) == 0,

2
video.go Normal file → Executable file
View file

@ -148,7 +148,7 @@ func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, err
}
// handleVideoSearch adapted from the Python `videoResults`, handles video search requests
func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query, safe, lang string, page int) {
func handleVideoSearch(w http.ResponseWriter, settings UserSettings, query string, page int) {
start := time.Now()
results := fetchVideoResults(query, settings.SafeSearch, settings.Language, page)