package conc import ( "crypto/tls" "models/internals/app/trace" "net/http" "reflect" "strings" "github.com/antchfx/htmlquery" "golang.org/x/net/html" "golang.org/x/net/html/charset" ) const ( totalChan = 10 ) var tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } var client = &http.Client{Transport: tr} // Item struct for html answer type Item struct { ID int64 IP string YeaLinkLink string CiscoLink string Error error Model string Firmware string Lang string Statuscode int } func (i *Item) getURL() *Item { resp, err := client.Get(i.YeaLinkLink) if err != nil { i.Error = err return i } if resp == nil { return i } if resp.StatusCode == 404 { resp, err = client.Get(i.CiscoLink) if err != nil { i.Error = err return i } if resp == nil { return i } } defer resp.Body.Close() r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) if err != nil { i.Error = err return i } doc, err := html.Parse(r) if err != nil { i.Error = err return i } nodes, err := htmlquery.Query(doc, "//title") if err != nil { i.Error = err return i } if nodes == nil { return i } // check T26 phone t26 := false td, _ := htmlquery.Query(doc, "//td[@contains(@id,'loginPhoneModel')]") if td != nil { if strings.Contains(htmlquery.InnerText(td), "T26P") { t26 = true } } i.Statuscode = resp.StatusCode model := handlerTitle(htmlquery.InnerText(nodes), t26) i.Model = model i.Lang = "ru" if model != "SPA303" { fnd, _ := htmlquery.Query(doc, "//script[contains(@src, '/Russian')]") if fnd == nil { fnd, _ = htmlquery.Query(doc, "//script[contains(@src, '11.Russian')]") } frm, _ := htmlquery.Query(doc, "//script[contains(@src, 'utility')]") if frm == nil { frm, _ = htmlquery.Query(doc, "//script[contains(@src, 'commonjs')]") } if frm != nil { src := htmlquery.SelectAttr(frm, "src") arrs := strings.Split(src, "?") if len(arrs) == 2 { i.Firmware = strings.TrimSpace(arrs[1]) } } else { frm, _ = htmlquery.Query(doc, "//script[contains(@src, '11.Russian')]") if frm != nil { src := htmlquery.SelectAttr(frm, "src") arrs := strings.Split(src, "?") if len(arrs) == 2 { i.Firmware = strings.TrimSpace(arrs[1]) } } } if fnd == nil { i.Lang = "en" } } return i } // Producer chan Item type Producer struct { out [totalChan]chan *Item } func (p *Producer) produce(i int, item *Item) { channel := i % totalChan p.out[channel] <- item.getURL() } // Concurency call goroutines func Concurency(items []*Item) []*Item { defer trace.UndoTrace(trace.MyTrace()) var handleItems []*Item m := [totalChan]chan *Item{} for i := range m { m[i] = make(chan *Item) } prod := Producer{out: m} for i, item := range items { go prod.produce(i, item) } cases := make([]reflect.SelectCase, len(prod.out)) for i, ch := range prod.out { cases[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(ch)} } ForLoop: for { _, value, _ := reflect.Select(cases) i := value.Elem().Interface().(Item) handleItems = append(handleItems, &i) if len(items) == len(handleItems) { break ForLoop } } return handleItems } func handlerTitle(title string, t26 bool) string { title = strings.Replace(title, "_", " ", -1) if strings.Contains(title, "502") { title = "" } if strings.Contains(title, "SPA303") { title = "SPA303" } if title == "Yealink Phone" { if t26 { title = "Yealink T26P Phone" } else { title = "Yealink T19 Phone" } } return title }