added libreY/X fix for pirate bay

files-thepiratebay.go

@@ -99,6 +99,11 @@ func (t *ThePirateBay) Search(query string, category string) ([]TorrentResult, e
 		})
 	}
 
+	// Check if the only result matches the specific "no results" result
+	if len(results) == 1 && (results[0].Title == "No results returned" || results[0].Magnet == "?xt=urn:btih:0000000000000000000000000000000000000000&dn=No-results-returned&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969%2Fannounce") {
+		return []TorrentResult{}, nil
+	}
+
 	return results, nil
 }
 

files.go

@@ -124,7 +124,6 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string,
 func fetchAndCacheFileResults(query, safe, lang string, page int) []TorrentResult {
 	sites := []TorrentSite{torrentGalaxy, nyaa, thePirateBay, rutor}
 	results := []TorrentResult{}
-	allErrors := true
 
 	for _, site := range sites {
 		if site == nil {
@@ -134,21 +133,12 @@ func fetchAndCacheFileResults(query, safe, lang string, page int) []TorrentResul
 		if err != nil {
 			continue
 		}
-		if len(res) > 0 {
-			allErrors = false
-		}
 		for _, r := range res {
 			r.Magnet = removeMagnetLink(r.Magnet) // Remove "magnet:", prehaps usless now?
 			results = append(results, r)
 		}
 	}
 
-	if allErrors || len(results) == 0 || results[len(results)-1].Title == "" || results[len(results)-1].Title == " " {
-		return []TorrentResult{
-			{Error: "Results are currently unavailable, sorry. Please try again later."},
-		}
-	}
-
 	// Cache the valid results
 	cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "file"}
 	resultsCache.Set(cacheKey, convertToSearchResults(results))

get-searchxng.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+const SEARX_SPACE_URL = "https://searx.space/data/instances.json"
+
+type SearXInstance struct {
+	URL      string `json:"url"`
+	Status   int    `json:"status_code"`
+	SSLGrade string `json:"grade"`
+}
+
+type SearXInstanceMetadata struct {
+	Timestamp int64 `json:"timestamp"`
+}
+
+type SearXInstanceResponse struct {
+	Metadata  SearXInstanceMetadata             `json:"metadata"`
+	Instances map[string]map[string]interface{} `json:"instances"`
+}
+
+var searxInstances []SearXInstance
+var searxInstanceLastFetched time.Time
+var searxInstanceFetchLock sync.Mutex
+
+var backupInstances = []SearXInstance{
+	{URL: "https://searx.ox2.fr/", Status: 200, SSLGrade: "A+"},
+	{URL: "https://search.datura.network/", Status: 200, SSLGrade: "A+"},
+	{URL: "https://searx.foss.family/", Status: 200, SSLGrade: "A+"},
+	// Add more backup instances as needed
+}
+
+func fetchSearXInstances() error {
+	resp, err := http.Get(SEARX_SPACE_URL)
+	if err != nil {
+		return fmt.Errorf("error fetching SearX instances: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	var instanceResp SearXInstanceResponse
+	if err := json.NewDecoder(resp.Body).Decode(&instanceResp); err != nil {
+		return fmt.Errorf("error decoding SearX instance response: %w", err)
+	}
+
+	searxInstances = make([]SearXInstance, 0)
+	for url, instanceData := range instanceResp.Instances {
+		if httpData, ok := instanceData["http"].(map[string]interface{}); ok {
+			if status, ok := httpData["status_code"].(float64); ok && int(status) == 200 {
+				if grade, ok := httpData["grade"].(string); ok && grade != "" {
+					searxInstances = append(searxInstances, SearXInstance{
+						URL:      url,
+						Status:   int(status),
+						SSLGrade: grade,
+					})
+				}
+			}
+		}
+	}
+
+	log.Printf("Fetched %d SearX instances: %+v", len(searxInstances), searxInstances)
+
+	searxInstanceLastFetched = time.Now()
+
+	return nil
+}
+
+func getRandomSearXInstance() (SearXInstance, error) {
+	searxInstanceFetchLock.Lock()
+	defer searxInstanceFetchLock.Unlock()
+
+	if searxInstances == nil || time.Since(searxInstanceLastFetched) > 24*time.Hour {
+		if err := fetchSearXInstances(); err != nil {
+			log.Printf("Error fetching instances, using backup instances: %v", err)
+			searxInstances = backupInstances
+		}
+	}
+
+	if len(searxInstances) == 0 {
+		return SearXInstance{}, fmt.Errorf("no available SearX instances")
+	}
+
+	for _, instance := range searxInstances {
+		if isInstanceValid(instance) {
+			log.Printf("Selected SearX instance: %+v", instance)
+			return instance, nil
+		}
+	}
+
+	return SearXInstance{}, fmt.Errorf("no valid SearX instances found")
+}
+
+func isInstanceValid(instance SearXInstance) bool {
+	searchURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", strings.TrimRight(instance.URL, "/"))
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	req, err := http.NewRequest("GET", searchURL, nil)
+	if err != nil {
+		log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err)
+		return false
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err)
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return true
+	} else {
+		log.Printf("Instance validation failed for URL: %s, StatusCode: %d", searchURL, resp.StatusCode)
+		return false
+	}
+}
+
+func main() {
+	instance, err := getRandomSearXInstance()
+	if err != nil {
+		log.Fatalf("Failed to get a SearX instance: %v", err)
+	}
+	fmt.Printf("Selected SearX instance: %s\n", instance.URL)
+}

run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug
+go run main.go images.go imageproxy.go video.go map.go text.go text-searchxng.go text-librex.go text-google.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug

text-librex.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+)
+
+const LIBREX_DOMAIN = "librex.antopie.org"
+
+type LibreXResult struct {
+	Title       string `json:"title"`
+	URL         string `json:"url"`
+	Description string `json:"description"`
+}
+
+type LibreXResponse []LibreXResult
+
+func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) {
+	// LibreX uses page starting from 0
+	searchURL := fmt.Sprintf("https://%s/api.php?q=%s&p=%d&t=0", LIBREX_DOMAIN, url.QueryEscape(query), page-1)
+
+	// User Agent generation
+	userAgent, err := GetUserAgent("librex-text-search")
+	if err != nil {
+		return nil, err
+	}
+
+	if debugMode {
+		log.Println("Generated User Agent (text):", userAgent)
+	}
+
+	req, err := http.NewRequest("GET", searchURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, logError("error making request to LibreX", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, logError("unexpected status code", fmt.Errorf("%d", resp.StatusCode))
+	}
+
+	var librexResp LibreXResponse
+	if err := json.NewDecoder(resp.Body).Decode(&librexResp); err != nil {
+		return nil, logError("error decoding LibreX response", err)
+	}
+
+	var results []TextSearchResult
+	for _, item := range librexResp {
+		result := TextSearchResult{
+			URL:         item.URL,
+			Header:      item.Title,
+			Description: item.Description,
+			Source:      "LibreX",
+		}
+
+		if debugMode {
+			log.Printf("LibreX result: %+v\n", result)
+		}
+
+		results = append(results, result)
+	}
+
+	return results, nil
+}
+
+func logError(message string, err error) error {
+	log.Printf("%s: %v", message, err)
+	return fmt.Errorf("%s: %w", message, err)
+}

text-searchxng.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"time"
+)
+
+type Instance struct {
+	URL      string `json:"url"`
+	Status   int    `json:"status"`
+	SSLGrade string `json:"ssl_grade"`
+}
+
+const searxInstancesURL = "https://searx.space/data/instances.json"
+
+func fetchInstances() ([]Instance, error) {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", searxInstancesURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var instances []Instance
+	err = json.Unmarshal(body, &instances)
+	if err != nil {
+		return nil, err
+	}
+
+	return instances, nil
+}
+
+func validateInstance(instance Instance) bool {
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL), nil)
+	if err != nil {
+		log.Printf("Error creating request for URL: %s, Error: %v", instance.URL, err)
+		return false
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		log.Printf("Error performing request for URL: %s, Error: %v", instance.URL, err)
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Printf("Instance validation failed for URL: %s, StatusCode: %d", instance.URL, resp.StatusCode)
+		return false
+	}
+
+	return true
+}
+
+func getValidInstance() (*Instance, error) {
+	instances, err := fetchInstances()
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch instances: %w", err)
+	}
+
+	for _, instance := range instances {
+		if validateInstance(instance) {
+			return &instance, nil
+		}
+	}
+
+	return nil, fmt.Errorf("no valid SearX instances found")
+}
+
+// func main() {
+// 	instance, err := getValidInstance()
+// 	if err != nil {
+// 		log.Fatalf("Failed to get a valid SearX instance: %v", err)
+// 	}
+
+// 	log.Printf("Selected SearX instance: %s", instance.URL)
+// }

text.go

@@ -135,8 +135,8 @@ func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int
 		Source string
 	}{
 		{PerformGoogleTextSearch, "Google"},
-		{PerformDuckDuckGoTextSearch, "DuckDuckGo"},
-		// {PerformQwantTextSearch, "Qwant"},
+		{PerformLibreXTextSearch, "LibreX"},
+		// {PerformSearXNGTextSearch, "SearXNG"},
 	}
 
 	wg.Add(len(searchFuncs))
@@ -180,10 +180,10 @@ func sourceOrder(source string) int {
 	switch source {
 	case "Google":
 		return 1
-	case "DuckDuckGo":
+	case "LibreX":
 		return 2
-	// case "Qwant":
-	// 	return 3
+	case "SearchXNG":
+		return 3
 	default:
 		return 4
 	}