Красимир обнови решението на 11.12.2014 13:32 (преди над 3 години)
+package main
+
+import (
+ "errors"
+ "io/ioutil"
+ "net/http"
+ "sync"
+ "time"
+)
+
+func handle(url string, callback func(string) bool) bool {
+ timeout := time.Duration(3 * time.Second)
+ client := http.Client{
+ Timeout: timeout,
+ }
+ resp, err := client.Get(url)
+ if err != nil {
+ return false
+ } else {
+ defer resp.Body.Close()
+ contents, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return false
+ }
+ if callback(string(contents)) {
+ return true
+ }
+ return false
+ }
+}
+
+type GoroLock struct {
+ free bool
+ sync.RWMutex
+}
+type UrlLock struct {
+ buffer []string
+ position int
+ count int
+ sync.RWMutex
+}
+
+func SeekAndDestroy(callback func(string) bool, chunkedUrlsToCheck <-chan []string, workersCount int) (string, error) {
+ if workersCount < 1 {
+ return "", errors.New("Bad things happen when you are not positive.")
+ }
+ if chunkedUrlsToCheck == nil {
+ return "", errors.New("Bad things happen when you cannot define yourself.")
+ }
+
+ endTime := time.Now().Add(15 * time.Second)
+ finish := make(chan string)
+ goroBuffer := []GoroLock{{true, *new(sync.RWMutex)}}
+ urlLock := UrlLock{make([]string, 100), 0, 0, *new(sync.RWMutex)}
+
+ for {
+ select {
+ case urls := <-chunkedUrlsToCheck:
+ for _, url := range urls {
+ foundFreeGoro := false // check if there are too many running goroutines
+ for _, goro := range goroBuffer {
+ goro.Lock()
+ if goro.free {
+ goro.free = false
+ goro.Unlock()
+ go func(url string) {
+ if handle(url, callback) {
+ finish <- url
+ return
+ } else {
+ checkForWaitingUrl: // You may say I'm a psycho, but I'm not the only one.
+ urlLock.Lock()
+ if (urlLock.count - urlLock.position) > 0 { // count will only grow up
+ urlLock.position++ // the position of the next non-handled url into the buffer
+ urlLock.Unlock()
+ if handle(urlLock.buffer[urlLock.position], callback) {
+ finish <- url
+ return
+ } else { // the goroutine won't be released until
+ goto checkForWaitingUrl // there's no left url into the buffer
+ }
+ }
+ urlLock.Unlock()
+ goro.Lock()
+ goro.free = true
+ defer goro.Unlock()
+ return // goroutine finished it's work, so there is place for one more goro...
+ }
+ }(url)
+ foundFreeGoro = true
+ break // found free goroutine for this url, no point of more iterations
+ } else {
+ goro.Unlock()
+ }
+ }
+ if !foundFreeGoro {
+ urlLock.Lock()
+ urlLock.count++
+ urlLock.buffer = append(urlLock.buffer, url)
+ urlLock.Unlock()
+ } else {
+ foundFreeGoro = false // everything is just fine for now
+ }
+ }
+ case foundUrl := <-finish:
+ return foundUrl, nil // found url that meets the requirements
+ default:
+ if time.Now().After(endTime) {
+ return "", errors.New("Bad things are(not) happening when nothing is happening.")
+ }
+ }
+ }
+}