From 606cae2dc96d08750910fbc92d4c03dae4c54a1c Mon Sep 17 00:00:00 2001 From: partisan Date: Sat, 8 Jun 2024 23:06:56 +0200 Subject: [PATCH] added libreY/X fix for pirate bay --- files-thepiratebay.go | 5 ++ files.go | 10 --- get-searchxng.go | 137 ++++++++++++++++++++++++++++++++++++++++++ run.sh | 2 +- text-librex.go | 79 ++++++++++++++++++++++++ text-searchxng.go | 98 ++++++++++++++++++++++++++++++ text.go | 10 +-- 7 files changed, 325 insertions(+), 16 deletions(-) create mode 100644 get-searchxng.go create mode 100644 text-librex.go create mode 100644 text-searchxng.go diff --git a/files-thepiratebay.go b/files-thepiratebay.go index e88d6f3..b98ee27 100644 --- a/files-thepiratebay.go +++ b/files-thepiratebay.go @@ -99,6 +99,11 @@ func (t *ThePirateBay) Search(query string, category string) ([]TorrentResult, e }) } + // Check if the only result matches the specific "no results" result + if len(results) == 1 && (results[0].Title == "No results returned" || results[0].Magnet == "?xt=urn:btih:0000000000000000000000000000000000000000&dn=No-results-returned&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80%2Fannounce&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969%2Fannounce&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969%2Fannounce") { + return []TorrentResult{}, nil + } + return results, nil } diff --git a/files.go b/files.go index 6e27afe..bce60b0 100644 --- a/files.go +++ b/files.go @@ -124,7 +124,6 @@ func getFileResultsFromCacheOrFetch(cacheKey CacheKey, query, safe, lang string, func fetchAndCacheFileResults(query, safe, lang string, page int) []TorrentResult { sites := []TorrentSite{torrentGalaxy, nyaa, thePirateBay, rutor} results := []TorrentResult{} - allErrors := true for _, site := range sites { if site == nil { @@ -134,21 +133,12 @@ func fetchAndCacheFileResults(query, safe, lang string, page int) []TorrentResul if err != nil { continue } - if len(res) > 0 { - allErrors = false - } for _, r := range res { r.Magnet = removeMagnetLink(r.Magnet) // Remove "magnet:", prehaps usless now? results = append(results, r) } } - if allErrors || len(results) == 0 || results[len(results)-1].Title == "" || results[len(results)-1].Title == " " { - return []TorrentResult{ - {Error: "Results are currently unavailable, sorry. Please try again later."}, - } - } - // Cache the valid results cacheKey := CacheKey{Query: query, Page: page, Safe: safe == "true", Lang: lang, Type: "file"} resultsCache.Set(cacheKey, convertToSearchResults(results)) diff --git a/get-searchxng.go b/get-searchxng.go new file mode 100644 index 0000000..cb88261 --- /dev/null +++ b/get-searchxng.go @@ -0,0 +1,137 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strings" + "sync" + "time" +) + +const SEARX_SPACE_URL = "https://searx.space/data/instances.json" + +type SearXInstance struct { + URL string `json:"url"` + Status int `json:"status_code"` + SSLGrade string `json:"grade"` +} + +type SearXInstanceMetadata struct { + Timestamp int64 `json:"timestamp"` +} + +type SearXInstanceResponse struct { + Metadata SearXInstanceMetadata `json:"metadata"` + Instances map[string]map[string]interface{} `json:"instances"` +} + +var searxInstances []SearXInstance +var searxInstanceLastFetched time.Time +var searxInstanceFetchLock sync.Mutex + +var backupInstances = []SearXInstance{ + {URL: "https://searx.ox2.fr/", Status: 200, SSLGrade: "A+"}, + {URL: "https://search.datura.network/", Status: 200, SSLGrade: "A+"}, + {URL: "https://searx.foss.family/", Status: 200, SSLGrade: "A+"}, + // Add more backup instances as needed +} + +func fetchSearXInstances() error { + resp, err := http.Get(SEARX_SPACE_URL) + if err != nil { + return fmt.Errorf("error fetching SearX instances: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var instanceResp SearXInstanceResponse + if err := json.NewDecoder(resp.Body).Decode(&instanceResp); err != nil { + return fmt.Errorf("error decoding SearX instance response: %w", err) + } + + searxInstances = make([]SearXInstance, 0) + for url, instanceData := range instanceResp.Instances { + if httpData, ok := instanceData["http"].(map[string]interface{}); ok { + if status, ok := httpData["status_code"].(float64); ok && int(status) == 200 { + if grade, ok := httpData["grade"].(string); ok && grade != "" { + searxInstances = append(searxInstances, SearXInstance{ + URL: url, + Status: int(status), + SSLGrade: grade, + }) + } + } + } + } + + log.Printf("Fetched %d SearX instances: %+v", len(searxInstances), searxInstances) + + searxInstanceLastFetched = time.Now() + + return nil +} + +func getRandomSearXInstance() (SearXInstance, error) { + searxInstanceFetchLock.Lock() + defer searxInstanceFetchLock.Unlock() + + if searxInstances == nil || time.Since(searxInstanceLastFetched) > 24*time.Hour { + if err := fetchSearXInstances(); err != nil { + log.Printf("Error fetching instances, using backup instances: %v", err) + searxInstances = backupInstances + } + } + + if len(searxInstances) == 0 { + return SearXInstance{}, fmt.Errorf("no available SearX instances") + } + + for _, instance := range searxInstances { + if isInstanceValid(instance) { + log.Printf("Selected SearX instance: %+v", instance) + return instance, nil + } + } + + return SearXInstance{}, fmt.Errorf("no valid SearX instances found") +} + +func isInstanceValid(instance SearXInstance) bool { + searchURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", strings.TrimRight(instance.URL, "/")) + client := &http.Client{ + Timeout: 10 * time.Second, + } + req, err := http.NewRequest("GET", searchURL, nil) + if err != nil { + log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err) + return false + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + + resp, err := client.Do(req) + if err != nil { + log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err) + return false + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + return true + } else { + log.Printf("Instance validation failed for URL: %s, StatusCode: %d", searchURL, resp.StatusCode) + return false + } +} + +func main() { + instance, err := getRandomSearXInstance() + if err != nil { + log.Fatalf("Failed to get a SearX instance: %v", err) + } + fmt.Printf("Selected SearX instance: %s\n", instance.URL) +} diff --git a/run.sh b/run.sh index 60eb68e..9b6d4d8 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug \ No newline at end of file +go run main.go images.go imageproxy.go video.go map.go text.go text-searchxng.go text-librex.go text-google.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug \ No newline at end of file diff --git a/text-librex.go b/text-librex.go new file mode 100644 index 0000000..450f20d --- /dev/null +++ b/text-librex.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" +) + +const LIBREX_DOMAIN = "librex.antopie.org" + +type LibreXResult struct { + Title string `json:"title"` + URL string `json:"url"` + Description string `json:"description"` +} + +type LibreXResponse []LibreXResult + +func PerformLibreXTextSearch(query, safe, lang string, page int) ([]TextSearchResult, error) { + // LibreX uses page starting from 0 + searchURL := fmt.Sprintf("https://%s/api.php?q=%s&p=%d&t=0", LIBREX_DOMAIN, url.QueryEscape(query), page-1) + + // User Agent generation + userAgent, err := GetUserAgent("librex-text-search") + if err != nil { + return nil, err + } + + if debugMode { + log.Println("Generated User Agent (text):", userAgent) + } + + req, err := http.NewRequest("GET", searchURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return nil, logError("error making request to LibreX", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, logError("unexpected status code", fmt.Errorf("%d", resp.StatusCode)) + } + + var librexResp LibreXResponse + if err := json.NewDecoder(resp.Body).Decode(&librexResp); err != nil { + return nil, logError("error decoding LibreX response", err) + } + + var results []TextSearchResult + for _, item := range librexResp { + result := TextSearchResult{ + URL: item.URL, + Header: item.Title, + Description: item.Description, + Source: "LibreX", + } + + if debugMode { + log.Printf("LibreX result: %+v\n", result) + } + + results = append(results, result) + } + + return results, nil +} + +func logError(message string, err error) error { + log.Printf("%s: %v", message, err) + return fmt.Errorf("%s: %w", message, err) +} diff --git a/text-searchxng.go b/text-searchxng.go new file mode 100644 index 0000000..3ed118f --- /dev/null +++ b/text-searchxng.go @@ -0,0 +1,98 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "time" +) + +type Instance struct { + URL string `json:"url"` + Status int `json:"status"` + SSLGrade string `json:"ssl_grade"` +} + +const searxInstancesURL = "https://searx.space/data/instances.json" + +func fetchInstances() ([]Instance, error) { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest("GET", searxInstancesURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var instances []Instance + err = json.Unmarshal(body, &instances) + if err != nil { + return nil, err + } + + return instances, nil +} + +func validateInstance(instance Instance) bool { + client := &http.Client{Timeout: 10 * time.Second} + req, err := http.NewRequest("GET", fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", instance.URL), nil) + if err != nil { + log.Printf("Error creating request for URL: %s, Error: %v", instance.URL, err) + return false + } + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + + resp, err := client.Do(req) + if err != nil { + log.Printf("Error performing request for URL: %s, Error: %v", instance.URL, err) + return false + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("Instance validation failed for URL: %s, StatusCode: %d", instance.URL, resp.StatusCode) + return false + } + + return true +} + +func getValidInstance() (*Instance, error) { + instances, err := fetchInstances() + if err != nil { + return nil, fmt.Errorf("failed to fetch instances: %w", err) + } + + for _, instance := range instances { + if validateInstance(instance) { + return &instance, nil + } + } + + return nil, fmt.Errorf("no valid SearX instances found") +} + +// func main() { +// instance, err := getValidInstance() +// if err != nil { +// log.Fatalf("Failed to get a valid SearX instance: %v", err) +// } + +// log.Printf("Selected SearX instance: %s", instance.URL) +// } diff --git a/text.go b/text.go index b65f57c..6845fab 100644 --- a/text.go +++ b/text.go @@ -135,8 +135,8 @@ func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int Source string }{ {PerformGoogleTextSearch, "Google"}, - {PerformDuckDuckGoTextSearch, "DuckDuckGo"}, - // {PerformQwantTextSearch, "Qwant"}, + {PerformLibreXTextSearch, "LibreX"}, + // {PerformSearXNGTextSearch, "SearXNG"}, } wg.Add(len(searchFuncs)) @@ -180,10 +180,10 @@ func sourceOrder(source string) int { switch source { case "Google": return 1 - case "DuckDuckGo": + case "LibreX": return 2 - // case "Qwant": - // return 3 + case "SearchXNG": + return 3 default: return 4 }