Merge pull request 'memoryfix' (#3) from work into main

Reviewed-on: #3

cache.go

@@ -1,13 +1,18 @@
-// common_cache.go
 package main
 
 import (
 	"fmt"
+	"log"
 	"sync"
 	"time"
+
+	"github.com/shirou/gopsutil/mem"
 )
 
-var resultsCache = NewResultsCache(6 * time.Hour) // Cache with 6-hour expiration
+var (
+	resultsCache   = NewResultsCache(6 * time.Hour) // Cache with 6-hour expiration
+	maxMemoryUsage = 90.0                           // Maximum memory usage in %
+)
 
 // SearchResult is a generic interface for all types of search results.
 type SearchResult interface{}
@@ -114,10 +119,14 @@ func (rc *ResultsCache) Get(key CacheKey) ([]SearchResult, bool) {
 func (rc *ResultsCache) Set(key CacheKey, results []SearchResult) {
 	rc.mu.Lock()
 	defer rc.mu.Unlock()
+
+	if _, exists := rc.results[rc.keyToString(key)]; !exists {
 		rc.results[rc.keyToString(key)] = CachedItem{
 			Results:    results,
 			StoredTime: time.Now(),
 		}
+		go rc.checkAndCleanCache()
+	}
 }
 
 // keyToString converts a CacheKey to a string representation.
@@ -125,7 +134,46 @@ func (rc *ResultsCache) keyToString(key CacheKey) string {
 	return fmt.Sprintf("%s|%d|%t|%s|%s", key.Query, key.Page, key.Safe, key.Lang, key.Type)
 }
 
-// Helper functions to convert between generic SearchResult and specific ImageSearchResult
+func (rc *ResultsCache) checkAndCleanCache() {
+	if rc.memoryUsage() > maxMemoryUsage {
+		rc.cleanOldestItems()
+	}
+}
+
+func (rc *ResultsCache) memoryUsage() float64 {
+	v, err := mem.VirtualMemory()
+	if err != nil {
+		log.Printf("Failed to get memory info: %v", err)
+		return 0
+	}
+
+	return v.UsedPercent
+}
+
+func (rc *ResultsCache) cleanOldestItems() {
+	rc.mu.Lock()
+	defer rc.mu.Unlock()
+
+	for rc.memoryUsage() > maxMemoryUsage {
+		var oldestKey string
+		var oldestTime time.Time = time.Now()
+
+		for key, item := range rc.results {
+			if item.StoredTime.Before(oldestTime) {
+				oldestTime = item.StoredTime
+				oldestKey = key
+			}
+		}
+
+		if oldestKey != "" {
+			delete(rc.results, oldestKey)
+			log.Printf("Removed oldest cache item: %s", oldestKey)
+		} else {
+			break
+		}
+	}
+}
+
 func convertToSearchResults(results interface{}) []SearchResult {
 	switch res := results.(type) {
 	case []TextSearchResult:

go.mod

@@ -14,7 +14,10 @@ require (
 	github.com/gobwas/ws v1.3.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 )

go.sum

@@ -8,6 +8,8 @@ github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93
 github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y=
 github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
 github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
@@ -20,7 +22,11 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
+github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -36,6 +42,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=