Merge branch 'develop'

web.go

@@ -5,6 +5,7 @@ import (
 	"flag"
 	"io/ioutil"
 	"net/http"
+	"strings"
 	"time"
 
 	"crawler.club/et"
@@ -58,13 +59,33 @@ func AddTaskHandler(w http.ResponseWriter, r *http.Request) {
 	rest.MustEncode(w, rest.RestMessage{"OK", k})
 }
 
+func DataHandler(w http.ResponseWriter, r *http.Request) {
+	glog.Infof("addr=%s  method=%s host=%s uri=%s",
+		r.RemoteAddr, r.Method, r.Host, r.RequestURI)
+	r.ParseForm()
+	peek := strings.ToLower(strings.TrimSpace(r.FormValue("peek")))
+	var ret string
+	var err error
+	if peek == "true" {
+		ret, err = storeQueue.Peek()
+	} else {
+		_, ret, err = storeQueue.Dequeue(-1)
+	}
+	if err != nil {
+		rest.MustEncode(w, rest.RestMessage{"ERROR", err.Error()})
+		return
+	}
+	w.Write([]byte(ret))
+}
+
 func web() {
 	if crawlQueue == nil || dedupStore == nil {
 		glog.Error("topics did not init, can't start web server")
 		return
 	}
 	http.Handle("/api/addtask", rest.WithLog(AddTaskHandler))
 	http.Handle("/api/status", rest.WithLog(StatusHandler))
+	http.Handle("/api/data", rest.WithLog(DataHandler))
 	glog.Info("rest server listen on", *addr)
 	glog.Error(http.ListenAndServe(*addr, nil))
 }