My golang program (url monitor) has a memory leak, it is finally killed by the kernel (oom). env:
$ go version go version go1.0.3 $ go env GOARCH="amd64" GOBIN="" GOCHAR="6" GOEXE="" GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread" GOHOSTARCH="amd64" GOHOSTOS="linux" GOOS="linux" GOPATH="/data/apps/go" GOROOT="/usr/local/go" GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" CGO_ENABLED="1"
the code:
package main import ( "bytes" "database/sql" "flag" "fmt" _ "github.com/Go-SQL-Driver/MySQL" "ijinshan.com/cfg" "log" "net" "net/http" "net/smtp" "os" "strconv" "strings" "sync" "time" ) var ( Log *log.Logger Conf cfg.KVConfig Debug bool CpuCore int HttpTransport = &http.Transport{ Dial: func(netw, addr string) (net.Conn, error) { deadline := time.Now().Add(30 * time.Second) c, err := net.DialTimeout(netw, addr, 20*time.Second) if err != nil { return nil, err } c.SetDeadline(deadline) return c, nil }, DisableKeepAlives: true, } HttpClient = &http.Client{ Transport: HttpTransport, } WG sync.WaitGroup ) const ( LogFileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND LogFileMode = 0644 LogFlag = log.LstdFlags | log.Lshortfile GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum, IFNULL(A.Website, ''), IFNULL(A.Descr, ''), IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''), IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '') FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200` HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名</td><td>tsid</td><td>章节</td><td>章节号</td><td>描述</td><td>videoid</td><td>网站</td><td>地址</td></tr>` HtmlTail = "</table>" ) type videoInfo struct { name string tsid uint chapter string chapterNum uint descr string videoId string website string androidWebUrl string iosWebUrl string androidUrl string androidUrl2 string iosUrl string } func init() { var ( confFile string err error ) // parse command argument:w flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path") flag.Parse() // read config if Conf, err = cfg.Read(confFile); err != nil { panic(fmt.Sprintf("Read config file \"%s\" failed (%s)", confFile, err.Error())) } // open log file file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode) if err != nil { panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"], err.Error())) } // init LOG Log = log.New(file, "", LogFlag) Debug = false i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32) if err != nil { panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"], err.Error())) } CpuCore = int(i) } func getHttpStatusCode(url string) int { if url == "" { return 200 } req, err := http.NewRequest("GET", url, nil) if err != nil { return 0 } req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17") req.Header.Add("Connection", "close") resp, err := HttpClient.Do(req) if err != nil { return 0 } defer resp.Body.Close() return resp.StatusCode } func sendMail(host, user, pwd, from, to, subject, body, mailType string) error { auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0]) cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType) msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s", to, from, user, subject, cntType, body) return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg)) } func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error { defer HttpTransport.CloseIdleConnections() db, err := sql.Open("mysql", Conf["weikan.mysql"]) if err != nil { return err } rows, err := db.Query(GET_VIDEO_SQL) if err != nil { db.Close() return err } for rows.Next() { video := &videoInfo{} err = rows.Scan(&video.name, &video.tsid, &video.chapter, &video.chapterNum, &video.website, &video.descr, &video.videoId, &video.androidWebUrl, &video.iosWebUrl, &video.androidUrl, &video.androidUrl2, &video.iosUrl) if err != nil { db.Close() return err } videoChan <- video WG.Add(1) } db.Close() // wait check url finish WG.Wait() // send mail for { if htmlBuf.Len() == 0 { Log.Print("no error found!!!!!!!!") break } Log.Print("found error !!!!!!!!") /* err := sendMail("smtp.gmail.com:587", "xxxx", "xxx", "xxx <xxx>", Conf["mail.to"], "xxxxx", HtmlHead+htmlBuf.String()+HtmlTail, "html") if err != nil { Log.Printf("sendMail failed (%s)", err.Error()) time.Sleep(10 * time.Second) continue } */ Log.Print("send mail") break } Log.Print("reset buf") htmlBuf.Reset() return nil } func checkUrl(videoChan chan *videoInfo, errChan chan string) { defer func() { if err := recover(); err != nil { Log.Print("rouintes failed : ", err) } }() for { video := <-videoChan ok := true errUrl := "" if code := getHttpStatusCode(video.androidWebUrl); code >= 400 { errUrl += fmt.Sprintf("%s (%d)<br />", video.androidWebUrl, code) ok = false } if code := getHttpStatusCode(video.iosWebUrl); code >= 400 { errUrl += fmt.Sprintf("%s (%d)<br />", video.iosWebUrl, code) ok = false } if code := getHttpStatusCode(video.androidUrl); code >= 400 { errUrl += fmt.Sprintf("%s (%d)<br />", video.androidUrl, code) ok = false } if code := getHttpStatusCode(video.androidUrl2); code >= 400 { errUrl += fmt.Sprintf("%s (%d)<br />", video.androidUrl2, code) ok = false } if code := getHttpStatusCode(video.iosUrl); code >= 400 { errUrl += fmt.Sprintf("%s (%d)<br />", video.iosUrl, code) ok = false } if !ok { errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td> <td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`, video.name, video.tsid, video.chapter, video.chapterNum, video.descr, video.videoId, video.website, errUrl) Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name, video.chapter, video.descr) } else { Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name, video.chapter, video.descr) WG.Done() } } } func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) { defer func() { if err := recover(); err != nil { Log.Print("rouintes failed : ", err) } }() for { html := <-errChan _, err := htmlBuf.WriteString(html) if err != nil { Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html, err.Error()) panic(err) } WG.Done() } } func main() { videoChan := make(chan *videoInfo, 100000) errChan := make(chan string, 100000) htmlBuf := &bytes.Buffer{} defer func() { if err := recover(); err != nil { Log.Print("rouintes failed : ", err) } }() // check url for i := 0; i < CpuCore; i++ { go checkUrl(videoChan, errChan) } // merge error string then send mail go mergeErr(errChan, htmlBuf) for { // get Video and LiveSrc video source if err := getVideos(videoChan, htmlBuf); err != nil { Log.Printf("getVideos failed (%s)", err.Error()) time.Sleep(10 * time.Second) continue } // time.Sleep(1 * time.Hour) } Log.Print("exit...") }
The code has four funcs :
getHttpStatusCode
free use of resources resp.Body.Close ()
Sendmail
I do not need to free the resource manually
mergeErr
concat string err using htmlBuf (* bytes.Buffer)
getVideos
First, it receives the URLs of the video, and then sends them to videoChan, after which it waits until all the routines have completed their checks. then sendmail and reset htmlBuf.
I can not find any resource that is needed for free, but.
$ top
shows:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND 6451 root 20 0 3946m 115m 2808 S 0.7 0.2 6:11.20 vsmonitor
VIRT and RES will grow ...
memory profiling:
(pprof) top Total: 10.8 MB 2.3 21.2% 21.2% 2.3 21.2% main.main 2.0 18.5% 39.8% 2.0 18.5% bufio.NewWriterSize 1.5 13.9% 53.7% 1.5 13.9% bufio.NewReaderSize 1.5 13.9% 67.6% 1.5 13.9% compress/flate.NewReader 0.5 4.6% 72.2% 0.5 4.6% net.newFD 0.5 4.6% 76.8% 0.5 4.6% net.sockaddrToTCP 0.5 4.6% 81.5% 4.5 41.7% net/http.(*Transport).getConn 0.5 4.6% 86.1% 2.5 23.2% net/http.(*persistConn).readLoop 0.5 4.6% 90.7% 0.5 4.6% net/textproto.(*Reader).ReadMIMEHeader 0.5 4.6% 95.4% 0.5 4.6% net/url.(*URL).ResolveReference