Search/get-searchxng.go

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"
	"time"
)

const SEARX_SPACE_URL = "https://searx.space/data/instances.json"

type SearXInstance struct {
	URL      string `json:"url"`
	Status   int    `json:"status_code"`
	SSLGrade string `json:"grade"`
}

type SearXInstanceMetadata struct {
	Timestamp int64 `json:"timestamp"`
}

type SearXInstanceResponse struct {
	Metadata  SearXInstanceMetadata             `json:"metadata"`
	Instances map[string]map[string]interface{} `json:"instances"`
}

var searxInstances []SearXInstance
var searxInstanceLastFetched time.Time
var searxInstanceFetchLock sync.Mutex

var backupInstances = []SearXInstance{
	{URL: "https://searx.ox2.fr/", Status: 200, SSLGrade: "A+"},
	{URL: "https://search.datura.network/", Status: 200, SSLGrade: "A+"},
	{URL: "https://searx.foss.family/", Status: 200, SSLGrade: "A+"},
	// Add more backup instances as needed
}

func fetchSearXInstances() error {
	resp, err := http.Get(SEARX_SPACE_URL)
	if err != nil {
		return fmt.Errorf("error fetching SearX instances: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
	}

	var instanceResp SearXInstanceResponse
	if err := json.NewDecoder(resp.Body).Decode(&instanceResp); err != nil {
		return fmt.Errorf("error decoding SearX instance response: %w", err)
	}

	searxInstances = make([]SearXInstance, 0)
	for url, instanceData := range instanceResp.Instances {
		if httpData, ok := instanceData["http"].(map[string]interface{}); ok {
			if status, ok := httpData["status_code"].(float64); ok && int(status) == 200 {
				if grade, ok := httpData["grade"].(string); ok && grade != "" {
					searxInstances = append(searxInstances, SearXInstance{
						URL:      url,
						Status:   int(status),
						SSLGrade: grade,
					})
				}
			}
		}
	}

	log.Printf("Fetched %d SearX instances: %+v", len(searxInstances), searxInstances)

	searxInstanceLastFetched = time.Now()

	return nil
}

func getRandomSearXInstance() (SearXInstance, error) {
	searxInstanceFetchLock.Lock()
	defer searxInstanceFetchLock.Unlock()

	if searxInstances == nil || time.Since(searxInstanceLastFetched) > 24*time.Hour {
		if err := fetchSearXInstances(); err != nil {
			log.Printf("Error fetching instances, using backup instances: %v", err)
			searxInstances = backupInstances
		}
	}

	if len(searxInstances) == 0 {
		return SearXInstance{}, fmt.Errorf("no available SearX instances")
	}

	for _, instance := range searxInstances {
		if isInstanceValid(instance) {
			log.Printf("Selected SearX instance: %+v", instance)
			return instance, nil
		}
	}

	return SearXInstance{}, fmt.Errorf("no valid SearX instances found")
}

func isInstanceValid(instance SearXInstance) bool {
	searchURL := fmt.Sprintf("%s/search?q=test&categories=general&language=en&safe_search=1&page=1&format=json", strings.TrimRight(instance.URL, "/"))
	client := &http.Client{
		Timeout: 10 * time.Second,
	}
	req, err := http.NewRequest("GET", searchURL, nil)
	if err != nil {
		log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err)
		return false
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")

	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Instance validation failed for URL: %s, Error: %v", searchURL, err)
		return false
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusOK {
		return true
	} else {
		log.Printf("Instance validation failed for URL: %s, StatusCode: %d", searchURL, resp.StatusCode)
		return false
	}
}

// func main() {
// 	instance, err := getRandomSearXInstance()
// 	if err != nil {
// 		log.Fatalf("Failed to get a SearX instance: %v", err)
// 	}
// 	fmt.Printf("Selected SearX instance: %s\n", instance.URL)
// }