<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/lair-framework/api-server/client"
	"github.com/lair-framework/go-lair"
	"github.com/lair-framework/go-nmap"
)

const (
	version  = "2.1.0"
	tool     = "nmap"
	osWeight = 50
	usage    = `
Parses an nmap XML file into a lair project.

Usage:
  drone-nmap [options] <id> <filename>
  export LAIR_ID=<id>; drone-nmap [options] <filename>
Options:
  -v              show version and exit
  -h              show usage and exit
  -k              allow insecure SSL connections
  -force-ports    disable data protection in the API server for excessive ports
  -limit-hosts    only import hosts that have listening ports
  -tags           a comma separated list of tags to add to every host that is imported
`
)

func buildProject(run *nmap.NmapRun, projectID string, tags []string) (*lair.Project, error) {
	project := &lair.Project{}
	project.ID = projectID
	project.Tool = tool
	project.Commands = append(project.Commands, lair.Command{Tool: tool, Command: run.Args})

	for _, h := range run.Hosts {
		host := &lair.Host{Tags: tags}
		if h.Status.State != "up" {
			continue
		}

		for _, address := range h.Addresses {
			switch {
			case address.AddrType == "ipv4":
				host.IPv4 = address.Addr
			case address.AddrType == "mac":
				host.MAC = address.Addr
			}
		}

		for _, hostname := range h.Hostnames {
			host.Hostnames = append(host.Hostnames, hostname.Name)
		}

		for _, p := range h.Ports {
			service := lair.Service{}
			service.Port = p.PortId
			service.Protocol = p.Protocol

			if p.State.State != "open" {
				continue
			}

			if p.Service.Name != "" {
				service.Service = p.Service.Name
				service.Product = "Unknown"
				if p.Service.Product != "" {
					service.Product = p.Service.Product
					if p.Service.Version != "" {
						service.Product += " " + p.Service.Version
					}
				}
			}

			for _, script := range p.Scripts {
				note := &lair.Note{Title: script.Id, Content: script.Output, LastModifiedBy: tool}
				service.Notes = append(service.Notes, *note)
			}

			host.Services = append(host.Services, service)
		}

		if len(h.Os.OsMatch) > 0 {
			os := lair.OS{}
			os.Tool = tool
			os.Weight = osWeight
			os.Fingerprint = h.Os.OsMatch[0].Name
			host.OS = os
		}

		project.Hosts = append(project.Hosts, *host)

	}

	return project, nil
}

func main() {
	showVersion := flag.Bool("v", false, "")
	insecureSSL := flag.Bool("k", false, "")
	forcePorts := flag.Bool("force-ports", false, "")
	limitHosts := flag.Bool("limit-hosts", false, "")
	tags := flag.String("tags", "", "")
	flag.Usage = func() {
		fmt.Println(usage)
	}
	flag.Parse()
	if *showVersion {
		log.Println(version)
		os.Exit(0)
	}
	lairURL := os.Getenv("LAIR_API_SERVER")
	if lairURL == "" {
		log.Fatal("Fatal: Missing LAIR_API_SERVER environment variable")
	}
	lairPID := os.Getenv("LAIR_ID")
	var filename string
	switch len(flag.Args()) {
	case 2:
		lairPID = flag.Arg(0)
		filename = flag.Arg(1)
	case 1:
		filename = flag.Arg(0)
	default:
		log.Fatal("Fatal: Missing required argument")
	}
	u, err := url.Parse(lairURL)
	if err != nil {
		log.Fatalf("Fatal: Error parsing LAIR_API_SERVER URL. Error %s", err.Error())
	}
	if u.User == nil {
		log.Fatal("Fatal: Missing username and/or password")
	}
	user := u.User.Username()
	pass, _ := u.User.Password()
	if user == "" || pass == "" {
		log.Fatal("Fatal: Missing username and/or password")
	}
	c, err := client.New(&client.COptions{
		User:               user,
		Password:           pass,
		Host:               u.Host,
		Scheme:             u.Scheme,
		InsecureSkipVerify: *insecureSSL,
	})
	if err != nil {
		log.Fatalf("Fatal: Error setting up client. Error %s", err.Error())
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Fatal: Could not open file. Error %s", err.Error())
	}
	hostTags := []string{}
	if *tags != "" {
		hostTags = strings.Split(*tags, ",")
	}
	nmapRun, err := nmap.Parse(data)
	if err != nil {
		log.Fatalf("Fatal: Error parsing nmap. Error %s", err.Error())
	}
	project, err := buildProject(nmapRun, lairPID, hostTags)
	if err != nil {
		log.Fatalf("Fatal: Error building project. Error %s", err.Error())
	}
	res, err := c.ImportProject(&client.DOptions{ForcePorts: *forcePorts, LimitHosts: *limitHosts}, project)
	if err != nil {
		log.Fatalf("Fatal: Unable to import project. Error %s", err.Error())
	}
	defer res.Body.Close()
	droneRes := &client.Response{}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Fatal: Error %s", err.Error())
	}
	if err := json.Unmarshal(body, droneRes); err != nil {
		log.Fatalf("Fatal: Could not unmarshal JSON. Error %s", err.Error())
	}
	if droneRes.Status == "Error" {
		log.Fatalf("Fatal: Import failed. Error %s", droneRes.Message)
	}
	log.Println("Success: Operation completed successfully")
}
<commit_msg>Update main.go<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"

	"github.com/lair-framework/api-server/client"
	"github.com/lair-framework/go-lair"
	"github.com/lair-framework/go-nmap"
)

const (
	version  = "2.1.0"
	tool     = "nmap"
	osWeight = 50
	usage    = `
Parses an nmap XML file into a lair project.

Usage:
  drone-nmap [options] <id> <filename>
  export LAIR_ID=<id>; drone-nmap [options] <filename>
Options:
  -v              show version and exit
  -h              show usage and exit
  -k              allow insecure SSL connections
  -force-ports    disable data protection in the API server for excessive ports
  -limit-hosts    only import hosts that have listening ports
  -tags           a comma separated list of tags to add to every host that is imported
`
)

func buildProject(run *nmap.NmapRun, projectID string, tags []string) (*lair.Project, error) {
	project := &lair.Project{}
	project.ID = projectID
	project.Tool = tool
	project.Commands = append(project.Commands, lair.Command{Tool: tool, Command: run.Args})

	for _, h := range run.Hosts {
		host := &lair.Host{Tags: tags}
		if h.Status.State != "up" {
			continue
		}

		for _, address := range h.Addresses {
			switch {
			case address.AddrType == "ipv4":
				host.IPv4 = address.Addr
			case address.AddrType == "mac":
				host.MAC = address.Addr
			}
		}

		for _, hostname := range h.Hostnames {
			host.Hostnames = append(host.Hostnames, hostname.Name)
		}

		for _, p := range h.Ports {
			service := lair.Service{}
			service.Port = p.PortId
			service.Protocol = p.Protocol

			if p.State.State != "open" {
				continue
			}

			if p.Service.Name != "" {
				service.Service = p.Service.Name
				service.Product = "Unknown"
				if p.Service.Product != "" {
					service.Product = p.Service.Product
					if p.Service.Version != "" {
						service.Product += " " + p.Service.Version
					}
				}
			}

			for _, script := range p.Scripts {
				note := &lair.Note{Title: script.Id, Content: script.Output, LastModifiedBy: tool}
				service.Notes = append(service.Notes, *note)
			}

			host.Services = append(host.Services, service)
		}

		if len(h.Os.OsMatch) > 0 {
			os := lair.OS{}
			os.Tool = tool
			os.Weight = osWeight
			os.Fingerprint = h.Os.OsMatch[0].Name
			host.OS = os
		}

		project.Hosts = append(project.Hosts, *host)

	}

	return project, nil
}

func main() {
	showVersion := flag.Bool("v", false, "")
	insecureSSL := flag.Bool("k", false, "")
	forcePorts := flag.Bool("force-ports", false, "")
	limitHosts := flag.Bool("limit-hosts", false, "")
	tags := flag.String("tags", "", "")
	flag.Usage = func() {
		fmt.Println(usage)
	}
	flag.Parse()
	if *showVersion {
		log.Println(version)
		os.Exit(0)
	}
	lairURL := os.Getenv("LAIR_API_SERVER")
	if lairURL == "" {
		log.Fatal("Fatal: Missing LAIR_API_SERVER environment variable")
	}
	lairPID := os.Getenv("LAIR_ID")
	if lairPID == "" {
		log.Fatal("Fatal: Missing LAIR_ID")
	}
	var filename string
	switch len(flag.Args()) {
	case 2:
		lairPID = flag.Arg(0)
		filename = flag.Arg(1)
	case 1:
		filename = flag.Arg(0)
	default:
		log.Fatal("Fatal: Missing required argument")
	}
	u, err := url.Parse(lairURL)
	if err != nil {
		log.Fatalf("Fatal: Error parsing LAIR_API_SERVER URL. Error %s", err.Error())
	}
	if u.User == nil {
		log.Fatal("Fatal: Missing username and/or password")
	}
	user := u.User.Username()
	pass, _ := u.User.Password()
	if user == "" || pass == "" {
		log.Fatal("Fatal: Missing username and/or password")
	}
	c, err := client.New(&client.COptions{
		User:               user,
		Password:           pass,
		Host:               u.Host,
		Scheme:             u.Scheme,
		InsecureSkipVerify: *insecureSSL,
	})
	if err != nil {
		log.Fatalf("Fatal: Error setting up client. Error %s", err.Error())
	}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalf("Fatal: Could not open file. Error %s", err.Error())
	}
	hostTags := []string{}
	if *tags != "" {
		hostTags = strings.Split(*tags, ",")
	}
	nmapRun, err := nmap.Parse(data)
	if err != nil {
		log.Fatalf("Fatal: Error parsing nmap. Error %s", err.Error())
	}
	project, err := buildProject(nmapRun, lairPID, hostTags)
	if err != nil {
		log.Fatalf("Fatal: Error building project. Error %s", err.Error())
	}
	res, err := c.ImportProject(&client.DOptions{ForcePorts: *forcePorts, LimitHosts: *limitHosts}, project)
	if err != nil {
		log.Fatalf("Fatal: Unable to import project. Error %s", err.Error())
	}
	defer res.Body.Close()
	droneRes := &client.Response{}
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Fatal: Error %s", err.Error())
	}
	if err := json.Unmarshal(body, droneRes); err != nil {
		log.Fatalf("Fatal: Could not unmarshal JSON. Error %s", err.Error())
	}
	if droneRes.Status == "Error" {
		log.Fatalf("Fatal: Import failed. Error %s", droneRes.Message)
	}
	log.Println("Success: Operation completed successfully")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"syscall"

	"github.com/BurntSushi/toml"
	"github.com/bitly/go-nsq"
	"github.com/garyburd/redigo/redis"
	"github.com/zenazn/goji/graceful"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
)

var (
	configPath = flag.String("config-file", "./config.toml", "path to qmd config file")
	config     Config

	producer *nsq.Producer
	consumer *nsq.Consumer
	redisDB  *redis.Pool
)

func main() {
	flag.Parse()
	fmt.Printf("Using config file from: %s\n", *configPath)

	var err error
	_, err = toml.DecodeFile(*configPath, &config)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println("Setting up producer")
	producer = nsq.NewProducer(config.QueueAddr, nsq.NewConfig())

	fmt.Println("Creating Redis connection pool")
	redisDB = newPool(config.RedisAddr)

	// Setup and start worker.
	fmt.Println("Creating worker")
	worker, err := NewWorker(config)
	if err != nil {
		log.Println(err)
		return
	}
	go worker.Run()

	// Http server
	w := web.New()
	w.Use(middleware.Logger)

	w.Get("/", ServiceRoot)
	w.Post("/", ServiceRoot)
	w.Get("/scripts", GetAllScripts)
	w.Put("/scripts", ReloadScripts)
	w.Post("/scripts/:name", RunScript)
	w.Get("/scripts/:name/logs", GetAllLogs)
	w.Get("/scripts/:name/logs/:id", GetLog)

	// Spin up the server with graceful hooks
	graceful.PreHook(func() {
		fmt.Println("Shutting down producer")
		producer.Stop()

		fmt.Println("Shutting down worker consumers")
		worker.Stop()

		fmt.Println("Closing Redis connections")
		redisDB.Close()
	})

	graceful.AddSignal(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	err = graceful.ListenAndServe(config.ListenOnAddr, w)
	if err != nil {
		log.Fatal(err)
	}
	graceful.Wait()
}

// func ExampleMiddleware(h http.Handler) http.Handler {
// 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 		log.Println("Request yooooooo")
// 		h.ServeHTTP(w, r)
// 	})
// }
<commit_msg>Added some commenting<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"syscall"

	"github.com/BurntSushi/toml"
	"github.com/bitly/go-nsq"
	"github.com/garyburd/redigo/redis"
	"github.com/zenazn/goji/graceful"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
)

var (
	configPath = flag.String("config-file", "./config.toml", "path to qmd config file")
	config     Config

	producer *nsq.Producer
	consumer *nsq.Consumer
	redisDB  *redis.Pool
)

func main() {
	flag.Parse()
	fmt.Printf("Using config file from: %s\n", *configPath)

	var err error
	_, err = toml.DecodeFile(*configPath, &config)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println("Setting up producer")
	producer = nsq.NewProducer(config.QueueAddr, nsq.NewConfig())

	fmt.Println("Creating Redis connection pool")
	redisDB = newPool(config.RedisAddr)

	// Setup and start worker.
	fmt.Println("Creating worker")
	worker, err := NewWorker(config)
	if err != nil {
		log.Println(err)
		return
	}
	go worker.Run()

	// Http server
	w := web.New()

	// Register middleware
	w.Use(middleware.Logger)

	// Register endpoints
	w.Get("/", ServiceRoot)
	w.Post("/", ServiceRoot)
	w.Get("/scripts", GetAllScripts)
	w.Put("/scripts", ReloadScripts)
	w.Post("/scripts/:name", RunScript)
	w.Get("/scripts/:name/logs", GetAllLogs)
	w.Get("/scripts/:name/logs/:id", GetLog)

	// Spin up the server with graceful hooks
	graceful.PreHook(func() {
		fmt.Println("Shutting down producer")
		producer.Stop()

		fmt.Println("Shutting down worker consumers")
		worker.Stop()

		fmt.Println("Closing Redis connections")
		redisDB.Close()
	})

	graceful.AddSignal(syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)

	err = graceful.ListenAndServe(config.ListenOnAddr, w)
	if err != nil {
		log.Fatal(err)
	}
	graceful.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"io/ioutil"
	"encoding/json"
	"strings"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/go-redis/redis"
	"strconv"
	"math"
	// "time"
)

type device struct {
	// Gps_num float32 `json:"gps_num"`
	// App string `json:"app"`
	// Gps_alt float32 `json:"gps_alt"`
	// Fmt_opt int `json:"fmt_opt"`
	// Device string `json:"device"`
	// S_d2 float32 `json:"s_d2"`
	S_d0 float32 `json:"s_d0"`
	// S_d1 float32 `json:"s_d1"`
	S_h0 float32 `json:"s_h0"`
	SiteName string `json:"SiteName"`
	// Gps_fix float32 `json:"gps_fix"`
	// Ver_app string `json:"ver_app"`
	Gps_lat float64 `json:"gps_lat"`
	S_t0 float32 `json:"s_t0"`
	Timestamp string `json:"timestamp"`
	Gps_lon float64 `json:"gps_lon"`
	// Date string `json:"date"`
	// Tick float32 `json:"tick"`
	Device_id string `json:"device_id"`
	// S_1 float32 `json:"s_1"`
	// S_0 float32 `json:"s_0"`
	// S_3 float32 `json:"s_3"`
	// S_2 float32 `json:"s_2"`
	// Ver_format string `json:"ver_format"`
	// Time string `json:"time"`
}

type airbox struct {
	Source string `json:"source"`
	Feeds []device `json:"feeds"`
	Version string `json:"version"`
	Num_of_records int `json:"num_of_records"`
}

type subscribeid struct {
	Device_id []string `json:"device_id"`
	Sitename []string `json:"sitename"`
}

var bot *linebot.Client
var airbox_json airbox
var lass_json airbox
var maps_json airbox
var all_device []device
var history_json subscribeid
var	client=redis.NewClient(&redis.Options{
		Addr:"hipposerver.ddns.net:6379",
		Password:"",
		DB:0,
	})

func main() {
	url := "https://data.lass-net.org/data/last-all-airbox.json"
	req, _ := http.NewRequest("GET", url, nil)
	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)
	errs := json.Unmarshal(body, &airbox_json)
	if errs != nil {
		fmt.Println(errs)
	}

	url = "https://data.lass-net.org/data/last-all-lass.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &lass_json)
	if errs != nil {
		fmt.Println(errs)
	}

	url = "https://data.lass-net.org/data/last-all-maps.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &maps_json)
	if errs != nil {
		fmt.Println(errs)
	}

	all_device=append(maps_json.Feeds,lass_json.Feeds...)
	all_device=append(all_device,airbox_json.Feeds...)

	url = "https://data.lass-net.org/data/airbox_list.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &history_json)
	if errs != nil {
		fmt.Println(errs)
	}
	// pushmessage()
	// fmt.Println(airbox_json)
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	// _,_=bot.PushMessage("U3617adbdd46283d7e859f36302f4f471", linebot.NewTextMessage("hi!")).Do()
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)

}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var txtmessage string
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText,"訂閱"){
					userID:=event.Source.UserID
					// pong, _ := client.Ping().Result()
					// txtmessage=pong
					for i:=0; i<len(history_json.Device_id); i++ {
						if strings.Contains(inText,strings.ToLower(history_json.Device_id[i]))||strings.Contains(inText,history_json.Sitename[i]) {
							val, err:=client.Get(history_json.Device_id[i]).Result()
							if err!=nil{
								if strings.Contains(inText,"取消"){
									txtmessage="你並沒有訂閱此ID。"
									break
								}
								client.Set(history_json.Device_id[i],userID,0)
								txtmessage="訂閱成功!"
								break
							}
							if strings.Contains(inText,"取消"){
								stringSlice:=strings.Split(val,",")
								if stringInSlice(userID,stringSlice){
									if len(stringSlice)==1{
										err=client.Del(history_json.Device_id[i]).Err()
										if err!=nil{
											txtmessage=err.Error()
											break
										}
										txtmessage="取消訂閱成功!"
										break
									}else{
										var s []string
										s = removeStringInSlice(stringSlice, userID)
										client.Set(history_json.Device_id[i],s,0)
										txtmessage="取消訂閱成功!"
										break
									}
								}else{
									txtmessage="你並沒有訂閱此ID。"
									break
								}
							}
							stringSlice:=strings.Split(val,",")
							if stringInSlice(userID,stringSlice){
								txtmessage="您已訂閱過此ID!"
								break
							} else{
								val=val+","+userID
								client.Set(history_json.Device_id[i],val,0)
								txtmessage="訂閱成功!"
								break
							}
						}
					}
				} else{
					for i:=0; i<len(all_device); i++ {
						if strings.Contains(inText,strings.ToLower(all_device[i].Device_id)) {
							txtmessage="Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
							break
						} else if len(all_device[i].SiteName)!=0 && strings.Contains(inText,all_device[i].SiteName){
							txtmessage="Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
							break
						}
					}
				}
				if len(txtmessage)==0{
					txtmessage="Sorry! No this device ID, please check again."
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			case *linebot.LocationMessage:
				var txtmessage string
				lat := message.Latitude
				lon := message.Longitude
				minD := math.MaxFloat64
				for i:=0; i<len(all_device); i++ {
					if all_device[i].Gps_lat<=lat+0.05 && all_device[i].Gps_lat>=lat-0.05 && all_device[i].Gps_lon<=lon+0.05 && all_device[i].Gps_lon>=lon-0.05{
						D:=distanceInKmBetweenEarthCoordinates(lat, lon, all_device[i].Gps_lat, all_device[i].Gps_lon)
						if D<minD{
							minD=D
							// txtmessage=strconv.FormatFloat(lat,'f',3,64)+","+strconv.FormatFloat(lon,'f',3,64)
							// txtmessage=strconv.FormatFloat(float64(minD),'f',3,64)
							txtmessage="Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
						}
					} else{
						continue
					}
				}
				if len(txtmessage)==0{
					txtmessage="很抱歉這附近1公里內沒有任何上線的AirBox。"
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}

func removeStringInSlice(s []string, r string) []string {
    for i, v := range s {
        if v == r {
            return append(s[:i], s[i+1:]...)
        }
    }
    return s
}

func degreesToRadians(degrees float64) float64 {
  return degrees * math.Pi / 180;
}

func distanceInKmBetweenEarthCoordinates(lat1 float64, lon1 float64, lat2 float64, lon2 float64) float64 {
  var earthRadiusKm = 6371.0;

  var dLat = degreesToRadians(lat2-lat1);
  var dLon = degreesToRadians(lon2-lon1);

  lat1 = degreesToRadians(lat1);
  lat2 = degreesToRadians(lat2);

  var a = math.Sin(dLat/2) * math.Sin(dLat/2) +
          math.Sin(dLon/2) * math.Sin(dLon/2) * math.Cos(lat1) * math.Cos(lat2);
  var c = 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a));
  return earthRadiusKm * c;
}

<commit_msg>修改訊息內容<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"io/ioutil"
	"encoding/json"
	"strings"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/go-redis/redis"
	"strconv"
	"math"
	// "time"
)

type device struct {
	// Gps_num float32 `json:"gps_num"`
	// App string `json:"app"`
	// Gps_alt float32 `json:"gps_alt"`
	// Fmt_opt int `json:"fmt_opt"`
	// Device string `json:"device"`
	// S_d2 float32 `json:"s_d2"`
	S_d0 float32 `json:"s_d0"`
	// S_d1 float32 `json:"s_d1"`
	S_h0 float32 `json:"s_h0"`
	SiteName string `json:"SiteName"`
	// Gps_fix float32 `json:"gps_fix"`
	// Ver_app string `json:"ver_app"`
	Gps_lat float64 `json:"gps_lat"`
	S_t0 float32 `json:"s_t0"`
	Timestamp string `json:"timestamp"`
	Gps_lon float64 `json:"gps_lon"`
	// Date string `json:"date"`
	// Tick float32 `json:"tick"`
	Device_id string `json:"device_id"`
	// S_1 float32 `json:"s_1"`
	// S_0 float32 `json:"s_0"`
	// S_3 float32 `json:"s_3"`
	// S_2 float32 `json:"s_2"`
	// Ver_format string `json:"ver_format"`
	// Time string `json:"time"`
}

type airbox struct {
	Source string `json:"source"`
	Feeds []device `json:"feeds"`
	Version string `json:"version"`
	Num_of_records int `json:"num_of_records"`
}

type subscribeid struct {
	Device_id []string `json:"device_id"`
	Sitename []string `json:"sitename"`
}

var bot *linebot.Client
var airbox_json airbox
var lass_json airbox
var maps_json airbox
var all_device []device
var history_json subscribeid
var	client=redis.NewClient(&redis.Options{
		Addr:"hipposerver.ddns.net:6379",
		Password:"",
		DB:0,
	})

func main() {
	url := "https://data.lass-net.org/data/last-all-airbox.json"
	req, _ := http.NewRequest("GET", url, nil)
	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)
	errs := json.Unmarshal(body, &airbox_json)
	if errs != nil {
		fmt.Println(errs)
	}

	url = "https://data.lass-net.org/data/last-all-lass.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &lass_json)
	if errs != nil {
		fmt.Println(errs)
	}

	url = "https://data.lass-net.org/data/last-all-maps.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &maps_json)
	if errs != nil {
		fmt.Println(errs)
	}

	all_device=append(maps_json.Feeds,lass_json.Feeds...)
	all_device=append(all_device,airbox_json.Feeds...)

	url = "https://data.lass-net.org/data/airbox_list.json"
	req, _ = http.NewRequest("GET", url, nil)
	res, _ = http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ = ioutil.ReadAll(res.Body)
	errs = json.Unmarshal(body, &history_json)
	if errs != nil {
		fmt.Println(errs)
	}
	// pushmessage()
	// fmt.Println(airbox_json)
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	// _,_=bot.PushMessage("U3617adbdd46283d7e859f36302f4f471", linebot.NewTextMessage("hi!")).Do()
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)

}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var txtmessage string
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText,"訂閱"){
					userID:=event.Source.UserID
					// pong, _ := client.Ping().Result()
					// txtmessage=pong
					for i:=0; i<len(history_json.Device_id); i++ {
						if strings.Contains(inText,strings.ToLower(history_json.Device_id[i]))||strings.Contains(inText,history_json.Sitename[i]) {
							val, err:=client.Get(history_json.Device_id[i]).Result()
							if err!=nil{
								if strings.Contains(inText,"取消"){
									txtmessage="你並沒有訂閱此ID。"
									break
								}
								client.Set(history_json.Device_id[i],userID,0)
								txtmessage="訂閱成功!"
								break
							}
							if strings.Contains(inText,"取消"){
								stringSlice:=strings.Split(val,",")
								if stringInSlice(userID,stringSlice){
									if len(stringSlice)==1{
										err=client.Del(history_json.Device_id[i]).Err()
										if err!=nil{
											txtmessage=err.Error()
											break
										}
										txtmessage="取消訂閱成功!"
										break
									}else{
										var s []string
										s = removeStringInSlice(stringSlice, userID)
										client.Set(history_json.Device_id[i],s,0)
										txtmessage="取消訂閱成功!"
										break
									}
								}else{
									txtmessage="你並沒有訂閱此ID。"
									break
								}
							}
							stringSlice:=strings.Split(val,",")
							if stringInSlice(userID,stringSlice){
								txtmessage="您已訂閱過此ID!"
								break
							} else{
								val=val+","+userID
								client.Set(history_json.Device_id[i],val,0)
								txtmessage="訂閱成功!"
								break
							}
						}
					}
				} else{
					for i:=0; i<len(all_device); i++ {
						if strings.Contains(inText,strings.ToLower(all_device[i].Device_id)) {
							txtmessage="Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
							break
						} else if len(all_device[i].SiteName)!=0 && strings.Contains(inText,all_device[i].SiteName){
							txtmessage="Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
							break
						}
					}
				}
				if len(txtmessage)==0{
					txtmessage="Sorry! No this device ID, please check again."
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			case *linebot.LocationMessage:
				var txtmessage string
				lat := message.Latitude
				lon := message.Longitude
				minD := math.MaxFloat64
				for i:=0; i<len(all_device); i++ {
					if all_device[i].Gps_lat<=lat+0.05 && all_device[i].Gps_lat>=lat-0.05 && all_device[i].Gps_lon<=lon+0.05 && all_device[i].Gps_lon>=lon-0.05{
						D:=distanceInKmBetweenEarthCoordinates(lat, lon, all_device[i].Gps_lat, all_device[i].Gps_lon)
						if D<minD{
							minD=D
							txtmessage="離您所提供的位置最近的AirBox資訊如以下所示：\n"
							txtmessage=txtmessage+"Device_id: "+all_device[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+all_device[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(all_device[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(all_device[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+all_device[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(all_device[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(all_device[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(all_device[i].S_t0),'f',0,64)
						}
					} else{
						continue
					}
				}
				if len(txtmessage)==0{
					txtmessage="很抱歉這附近1公里內沒有任何上線的AirBox。"
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}

func removeStringInSlice(s []string, r string) []string {
    for i, v := range s {
        if v == r {
            return append(s[:i], s[i+1:]...)
        }
    }
    return s
}

func degreesToRadians(degrees float64) float64 {
  return degrees * math.Pi / 180;
}

func distanceInKmBetweenEarthCoordinates(lat1 float64, lon1 float64, lat2 float64, lon2 float64) float64 {
  var earthRadiusKm = 6371.0;

  var dLat = degreesToRadians(lat2-lat1);
  var dLon = degreesToRadians(lon2-lon1);

  lat1 = degreesToRadians(lat1);
  lat2 = degreesToRadians(lat2);

  var a = math.Sin(dLat/2) * math.Sin(dLat/2) +
          math.Sin(dLon/2) * math.Sin(dLon/2) * math.Cos(lat1) * math.Cos(lat2);
  var c = 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a));
  return earthRadiusKm * c;
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "net/http/pprof"

	_ "github.com/go-sql-driver/mysql"
	"github.com/reusee/catch"

	"github.com/PuerkitoBio/goquery"
	"github.com/bradfitz/http2"
)

var (
	pt          = fmt.Printf
	sp          = fmt.Sprintf
	logFilePath = flag.String("log", "", "log file path")
	ce          = catch.PkgChecker("bilibili")
	ct          = catch.Catch
)

func main() {
	flag.Parse()
	defer func() {
		if p := recover(); p != nil && *logFilePath != "" {
			ioutil.WriteFile(*logFilePath, []byte(fmt.Sprintf("%v", p)), 0644)
		}
	}()

	db, err := sql.Open("mysql", "root@unix(/var/run/mysqld/mysqld.sock)/bilibili?tokudb_commit_sync=off")
	ce(err, "connect to db")

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS video (
		id INTEGER PRIMARY KEY,
		title TEXT,
		view INTEGER NOT NULL DEFAULT 0,
		last_visit INTEGER,
		image TEXT,
		added INTEGER DEFAULT 0,
		uid CHAR(64)
		) engine=TokuDB`)
	ce(err, "create table")

	get := func(url string) []byte {
		retryCount := 8
	retry:
		client := http.Client{
			Timeout: time.Second * 16,
		}
		pt("get %s\n", url)
		resp, err := client.Get(url)
		if err != nil {
			if retryCount > 0 {
				retryCount--
				time.Sleep(time.Second * 3)
				goto retry
			} else {
				ce(err, sp("get %s", url))
			}
		}
		defer resp.Body.Close()
		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			if retryCount > 0 {
				retryCount--
				time.Sleep(time.Second * 3)
				goto retry
			} else {
				ce(err, sp("read %s", url))
			}
		}
		return content
	}

	bytesToDoc := func(bs []byte) *goquery.Document {
		doc, err := goquery.NewDocumentFromReader(bytes.NewReader(bs))
		ce(err, "get document")
		return doc
	}

	collectFollowers := func() []string {

	getMaxPage:
		doc := bytesToDoc(get("http://space.bilibili.com/19415/follow.html?page=1"))
		maxPage := doc.Find("ul.page li").Length()
		if maxPage == 0 {
			goto getMaxPage
		}
		users := []string{}
		collectUser := func(url string) {
		getDoc:
			doc := bytesToDoc(get(url))
			entries := doc.Find("div.fanslist ul li div.t a[card]")
			if entries.Length() == 0 {
				goto getDoc
			}
			entries.Each(func(n int, se *goquery.Selection) {
				href, ok := se.Attr("href")
				if !ok {
					panic("invalid entry")
				}
				uid := href[strings.LastIndex(href, "/")+1:]
				users = append(users, uid)
			})
		}
		for i := 1; i <= maxPage; i++ {
			collectUser(sp("http://space.bilibili.com/19415/follow.html?page=%d", i))
		}
		return users
	}

	collectVideo := func(uid string, page int) int {
		url := sp("http://space.bilibili.com/space?uid=%s&page=%d", uid, page)
		doc := bytesToDoc(get(url))
		entries := doc.Find("div.main_list ul li")
		count := 0
		entries.Each(func(n int, se *goquery.Selection) {
			titleSe := se.Find("a.title")
			title := titleSe.Text()
			href, ok := titleSe.Attr("href")
			if !ok || len(title) == 0 {
				panic("invalid entry")
			}
			idStr := href[strings.LastIndex(href, "av")+2:]
			idStr = idStr[:len(idStr)-1]
			id, err := strconv.Atoi(idStr)
			imgSe := se.Find("a img")
			image, ok := imgSe.Attr("src")
			if !ok {
				panic("no image")
			}
			_, err = db.Exec(`INSERT IGNORE INTO video (id, title, image, added, uid)
				VALUES (?, ?, ?, ?, ?)`, id, title, image, time.Now().Unix(), uid)
			ce(err, "insert")
			count++
		})
		return count
	}

	collectFollowed := func() {
		t0 := time.Now()
		users := collectFollowers()
		for _, uid := range users {
			page := 1
			for page < 50 {
				count := collectVideo(uid, page)
				if count == 0 {
					break
				}
				page++
			}
		}
		pt("collected in %v\n", time.Now().Sub(t0))
	}

	collectHottest := func() {
		urls := []string{
			"http://www.bilibili.com/video/bagumi_offical_1.html#!order=hot&page=1", // 官方延伸所有新投稿
		}
		end := time.Now()
		start := end.AddDate(0, 0, -7)
		rangeStr := sp("%4d-%02d-%02d~%4d-%02d-%02d", start.Year(), start.Month(), start.Day(),
			end.Year(), end.Month(), end.Day())
		urls = append(urls, sp("http://www.bilibili.com/list/damku-29-1-%s.html", rangeStr)) // 三次元音乐弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-17-1-%s.html", rangeStr)) // 单机联机弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-37-1-%s.html", rangeStr)) // 纪录片弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-51-1-%s.html", rangeStr)) // 动画资讯弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-98-1-%s.html", rangeStr)) // 机械弹幕排序
		for _, url := range urls {
			doc := bytesToDoc(get(url))
			entries := doc.Find("ul.vd-list li")
			entries.Each(func(i int, se *goquery.Selection) {
				titleSe := se.Find("a.title")
				title := titleSe.Text()
				href, ok := titleSe.Attr("href")
				if !ok || len(title) == 0 {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				idStr := href[strings.LastIndex(href, "av")+2:]
				idStr = idStr[:len(idStr)-1]
				id, err := strconv.Atoi(idStr)
				if err != nil {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				imgSe := se.Find("a img")
				image, ok := imgSe.Attr("src")
				if !ok {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				//pt("%d %s %s\n", id, title, image)
				_, err = db.Exec(`INSERT IGNORE INTO video (id, title, image, added)
					VALUES (?, ?, ?, ?)`, id, title, image, time.Now().Unix())
				ce(err, "insert")
			})
		}
	}

	go func() {
		for {
			func() {
				defer recover()
				collectHottest()
				collectFollowed()
			}()
			time.Sleep(time.Minute * 5)
		}
	}()

	root := "./web"
	if len(os.Args) > 1 {
		root = os.Args[1]
	}
	root, err = filepath.Abs(root)
	ce(err, "get web root dir")
	pt("web root %s\n", root)
	http.Handle("/", http.FileServer(http.Dir(root)))

	http.HandleFunc("/newest.json", func(w http.ResponseWriter, req *http.Request) {
		videos := []Video{}
		rows, err := db.Query(`SELECT id, title, image FROM video
			WHERE view < 1
			ORDER BY id DESC, added DESC LIMIT 50`)
		ce(err, "query")
		for rows.Next() {
			var video Video
			ce(rows.Scan(&video.Id, &video.Title, &video.Image), "scan")
			videos = append(videos, video)
		}
		ce(rows.Err(), "rows")
		bs, err := json.Marshal(videos)
		ce(err, "marshal")
		w.Write(bs)
	})

	http.HandleFunc("/recently.json", func(w http.ResponseWriter, req *http.Request) {
		videos := []Video{}
		rows, err := db.Query(`SELECT id, title, image FROM video
			WHERE last_visit IS NOT NULL
			ORDER BY last_visit DESC LIMIT 20`)
		ce(err, "query")
		for rows.Next() {
			var video Video
			ce(rows.Scan(&video.Id, &video.Title, &video.Image), "scan")
			videos = append(videos, video)
		}
		ce(rows.Err(), "rows")
		bs, err := json.Marshal(videos)
		ce(err, "marshal")
		w.Write(bs)
	})

	http.HandleFunc("/go", func(w http.ResponseWriter, req *http.Request) {
		idStr := req.FormValue("id")
		id, err := strconv.Atoi(idStr)
		ce(err, "parse id")
		_, err = db.Exec(`UPDATE video SET
			view = view + 1,
			last_visit = ?
			WHERE id = ?`, id, time.Now().Unix())
		ce(err, "update")
		http.Redirect(w, req, sp("http://www.bilibili.com/video/av%d", id), 307)
	})

	http.HandleFunc("/mark", func(w http.ResponseWriter, req *http.Request) {
		idStr := req.FormValue("id")
		id, err := strconv.Atoi(idStr)
		ce(err, "parse id")
		_, err = db.Exec(`UPDATE video SET
			view = view + 1,
			last_visit = ?
			WHERE id = ?`, id, time.Now().Unix())
		ce(err, "update")
		bs, err := json.Marshal(struct{ Ok bool }{true})
		ce(err, "marshal json")
		w.Write(bs)
	})

	pt("starting http server\n")
	server := http.Server{
		Addr: ":19870",
	}
	http2.ConfigureServer(&server, nil)
	err = server.ListenAndServe()
	ce(err, "start http server")
}

type Video struct {
	Title string `db:"title" json:"title"`
	Id    int    `db:"id" json:"id"`
	View  int    `db:"view" json:"view"`
	Image string `db:"image" json:"image"`
}
<commit_msg>do not use INSERT IGNORE<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	_ "net/http/pprof"

	_ "github.com/go-sql-driver/mysql"
	"github.com/reusee/catch"

	"github.com/PuerkitoBio/goquery"
	"github.com/bradfitz/http2"
)

var (
	pt          = fmt.Printf
	sp          = fmt.Sprintf
	logFilePath = flag.String("log", "", "log file path")
	ce          = catch.PkgChecker("bilibili")
	ct          = catch.Catch
)

func main() {
	flag.Parse()
	defer func() {
		if p := recover(); p != nil && *logFilePath != "" {
			ioutil.WriteFile(*logFilePath, []byte(fmt.Sprintf("%v", p)), 0644)
		}
	}()

	db, err := sql.Open("mysql", "root@unix(/var/run/mysqld/mysqld.sock)/bilibili?tokudb_commit_sync=off")
	ce(err, "connect to db")

	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS video (
		id INTEGER PRIMARY KEY,
		title TEXT,
		view INTEGER NOT NULL DEFAULT 0,
		last_visit INTEGER,
		image TEXT,
		added INTEGER DEFAULT 0,
		uid CHAR(64)
		) engine=TokuDB`)
	ce(err, "create table")

	get := func(url string) []byte {
		retryCount := 8
	retry:
		client := http.Client{
			Timeout: time.Second * 16,
		}
		pt("get %s\n", url)
		resp, err := client.Get(url)
		if err != nil {
			if retryCount > 0 {
				retryCount--
				time.Sleep(time.Second * 3)
				goto retry
			} else {
				ce(err, sp("get %s", url))
			}
		}
		defer resp.Body.Close()
		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			if retryCount > 0 {
				retryCount--
				time.Sleep(time.Second * 3)
				goto retry
			} else {
				ce(err, sp("read %s", url))
			}
		}
		return content
	}

	bytesToDoc := func(bs []byte) *goquery.Document {
		doc, err := goquery.NewDocumentFromReader(bytes.NewReader(bs))
		ce(err, "get document")
		return doc
	}

	collectFollowers := func() []string {

	getMaxPage:
		doc := bytesToDoc(get("http://space.bilibili.com/19415/follow.html?page=1"))
		maxPage := doc.Find("ul.page li").Length()
		if maxPage == 0 {
			goto getMaxPage
		}
		users := []string{}
		collectUser := func(url string) {
		getDoc:
			doc := bytesToDoc(get(url))
			entries := doc.Find("div.fanslist ul li div.t a[card]")
			if entries.Length() == 0 {
				goto getDoc
			}
			entries.Each(func(n int, se *goquery.Selection) {
				href, ok := se.Attr("href")
				if !ok {
					panic("invalid entry")
				}
				uid := href[strings.LastIndex(href, "/")+1:]
				users = append(users, uid)
			})
		}
		for i := 1; i <= maxPage; i++ {
			collectUser(sp("http://space.bilibili.com/19415/follow.html?page=%d", i))
		}
		return users
	}

	collectVideo := func(uid string, page int) int {
		url := sp("http://space.bilibili.com/space?uid=%s&page=%d", uid, page)
		doc := bytesToDoc(get(url))
		entries := doc.Find("div.main_list ul li")
		count := 0
		entries.Each(func(n int, se *goquery.Selection) {
			titleSe := se.Find("a.title")
			title := titleSe.Text()
			href, ok := titleSe.Attr("href")
			if !ok || len(title) == 0 {
				panic("invalid entry")
			}
			idStr := href[strings.LastIndex(href, "av")+2:]
			idStr = idStr[:len(idStr)-1]
			id, err := strconv.Atoi(idStr)
			imgSe := se.Find("a img")
			image, ok := imgSe.Attr("src")
			if !ok {
				panic("no image")
			}
			_, err = db.Exec(`INSERT INTO video (id, title, image, added, uid)
				VALUES (?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id`, id, title, image, time.Now().Unix(), uid)
			ce(err, "insert")
			count++
		})
		return count
	}

	collectFollowed := func() {
		t0 := time.Now()
		users := collectFollowers()
		for _, uid := range users {
			page := 1
			for page < 50 {
				count := collectVideo(uid, page)
				if count == 0 {
					break
				}
				page++
			}
		}
		pt("collected in %v\n", time.Now().Sub(t0))
	}

	collectHottest := func() {
		urls := []string{
			"http://www.bilibili.com/video/bagumi_offical_1.html#!order=hot&page=1", // 官方延伸所有新投稿
		}
		end := time.Now()
		start := end.AddDate(0, 0, -7)
		rangeStr := sp("%4d-%02d-%02d~%4d-%02d-%02d", start.Year(), start.Month(), start.Day(),
			end.Year(), end.Month(), end.Day())
		urls = append(urls, sp("http://www.bilibili.com/list/damku-29-1-%s.html", rangeStr)) // 三次元音乐弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-17-1-%s.html", rangeStr)) // 单机联机弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-37-1-%s.html", rangeStr)) // 纪录片弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-51-1-%s.html", rangeStr)) // 动画资讯弹幕排序
		urls = append(urls, sp("http://www.bilibili.com/list/damku-98-1-%s.html", rangeStr)) // 机械弹幕排序
		for _, url := range urls {
			doc := bytesToDoc(get(url))
			entries := doc.Find("ul.vd-list li")
			entries.Each(func(i int, se *goquery.Selection) {
				titleSe := se.Find("a.title")
				title := titleSe.Text()
				href, ok := titleSe.Attr("href")
				if !ok || len(title) == 0 {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				idStr := href[strings.LastIndex(href, "av")+2:]
				idStr = idStr[:len(idStr)-1]
				id, err := strconv.Atoi(idStr)
				if err != nil {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				imgSe := se.Find("a img")
				image, ok := imgSe.Attr("src")
				if !ok {
					panic(sp("invalid entry in %s # %d", url, i))
				}
				//pt("%d %s %s\n", id, title, image)
				_, err = db.Exec(`INSERT INTO video (id, title, image, added)
					VALUES (?, ?, ?, ?) ON DUPLICATE KEY UPDATE id=id`, id, title, image, time.Now().Unix())
				ce(err, "insert")
			})
		}
	}

	go func() {
		for {
			func() {
				defer recover()
				collectHottest()
				collectFollowed()
			}()
			time.Sleep(time.Minute * 5)
		}
	}()

	root := "./web"
	if len(os.Args) > 1 {
		root = os.Args[1]
	}
	root, err = filepath.Abs(root)
	ce(err, "get web root dir")
	pt("web root %s\n", root)
	http.Handle("/", http.FileServer(http.Dir(root)))

	http.HandleFunc("/newest.json", func(w http.ResponseWriter, req *http.Request) {
		videos := []Video{}
		rows, err := db.Query(`SELECT id, title, image FROM video
			WHERE view < 1
			ORDER BY id DESC, added DESC LIMIT 50`)
		ce(err, "query")
		for rows.Next() {
			var video Video
			ce(rows.Scan(&video.Id, &video.Title, &video.Image), "scan")
			videos = append(videos, video)
		}
		ce(rows.Err(), "rows")
		bs, err := json.Marshal(videos)
		ce(err, "marshal")
		w.Write(bs)
	})

	http.HandleFunc("/recently.json", func(w http.ResponseWriter, req *http.Request) {
		videos := []Video{}
		rows, err := db.Query(`SELECT id, title, image FROM video
			WHERE last_visit IS NOT NULL
			ORDER BY last_visit DESC LIMIT 20`)
		ce(err, "query")
		for rows.Next() {
			var video Video
			ce(rows.Scan(&video.Id, &video.Title, &video.Image), "scan")
			videos = append(videos, video)
		}
		ce(rows.Err(), "rows")
		bs, err := json.Marshal(videos)
		ce(err, "marshal")
		w.Write(bs)
	})

	http.HandleFunc("/go", func(w http.ResponseWriter, req *http.Request) {
		idStr := req.FormValue("id")
		id, err := strconv.Atoi(idStr)
		ce(err, "parse id")
		_, err = db.Exec(`UPDATE video SET
			view = view + 1,
			last_visit = ?
			WHERE id = ?`, time.Now().Unix(), id)
		ce(err, "update")
		http.Redirect(w, req, sp("http://www.bilibili.com/video/av%d", id), 307)
	})

	http.HandleFunc("/mark", func(w http.ResponseWriter, req *http.Request) {
		idStr := req.FormValue("id")
		id, err := strconv.Atoi(idStr)
		ce(err, "parse id")
		_, err = db.Exec(`UPDATE video SET
			view = view + 1,
			last_visit = ?
			WHERE id = ?`, time.Now().Unix(), id)
		ce(err, "update")
		bs, err := json.Marshal(struct{ Ok bool }{true})
		ce(err, "marshal json")
		w.Write(bs)
	})

	pt("starting http server\n")
	server := http.Server{
		Addr: ":19870",
	}
	http2.ConfigureServer(&server, nil)
	err = server.ListenAndServe()
	ce(err, "start http server")
}

type Video struct {
	Title string `db:"title" json:"title"`
	Id    int    `db:"id" json:"id"`
	View  int    `db:"view" json:"view"`
	Image string `db:"image" json:"image"`
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	nlog "github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/support"
	"github.com/ninjasphere/sphere-go-led-controller/model"
	"github.com/ninjasphere/sphere-go-led-controller/util"
	"github.com/paypal/gatt"
)

const WirelessNetworkInterface = "wlan0"

// consider the wifi to be invalid after this timeout
const WirelessStaleTimeout = time.Second * 30 // FIXME: INCREASE THIS. a few minutes at least when not in testing.

var firewallHook = flag.Bool("firewall-hook", false, "Sets up the firewall based on configuration options, and nothing else.")
var factoryReset = false
var imagesDir = ""

// factoryReset - we can't use anything that requires MQTT in this mode

var logger = nlog.GetLogger("sphere-setup")

func main() {
	// ap0 adhoc/hostap management
	flag.BoolVar(&factoryReset, "factory-reset", false, "Run in factory reset mode.")
	flag.StringVar(&imagesDir, "images", "/opt/ninjablocks/drivers/sphere-go-led-controller/images", "The location of the images directory.")

	go func() {
		support.WaitUntilSignal()
	}()

	config := LoadConfig("/etc/opt/ninja/setup-assistant.conf")
	apManager := NewAccessPointManager(config)

	flag.Parse()

	if *firewallHook {
		logger.Debugf("Setting ip firewall rules...")
		apManager.SetupFirewall()
		return
	}

	util.SetImageDir(imagesDir)
	var pairing_ui ConsolePairingUI
	var controlChecker *ControlChecker

	restartHeartbeat := false

	startResetMonitor(func(m *model.ResetMode) {
		if pairing_ui == nil || controlChecker == nil {
			return
		}
		if m.Mode == "none" {
			if restartHeartbeat {
				// only restart the heartbeat if we stopped it previously
				controlChecker.StartHeartbeat()
			}
		} else {
			restartHeartbeat = controlChecker.StopHeartbeat()
			pairing_ui.DisplayResetMode(m)
		}
	})

	apManager.WriteAPConfig()
	if config.Wireless_Host.Always_Active {
		apManager.StartHostAP()
	} else {
		apManager.StopHostAP()
	}

	// wlan0 client management
	wifi_manager, err := NewWifiManager(WirelessNetworkInterface)
	if err != nil {
		log.Fatal("Could not setup manager for wlan0, does the interface exist?: %v", err)
	}
	defer wifi_manager.Cleanup()

	// When in reset mode, this will talk to the led matrix directly,
	// otherwise, it will use sphere-go-led-controller via mqtt.
	pairing_ui, err = NewPairingUI()
	if err != nil {
		log.Fatal("Could not setup ninja connection")
	}

	// This name is sent in the BLE advertising packet,
	// and is used by the phone to see that this is a
	// sphere, and if it is in factory reset mode.
	serviceName := "ninjasphere"
	if factoryReset {
		serviceName = "resetsphere"
	}

	srv := &gatt.Server{
		Name: serviceName,
		Connect: func(c gatt.Conn) {
			logger.Infof("BLE Connect")
			pairing_ui.DisplayIcon("ble-connected.gif")
		},
		Disconnect: func(c gatt.Conn) {
			logger.Infof("BLE Disconnect")
			pairing_ui.DisplayIcon("ble-disconnected.gif")
		},
		StateChange: func(state string) {
			logger.Infof("BLE State Change: %s", state)
		},
	}

	var conn *ninja.Connection

	if !factoryReset {

		conn, err = ninja.Connect("sphere-setup-assistant-updates")

		if err != nil {
			logger.FatalErrorf(err, "Failed to connect to mqtt")
		}
	}

	// start by registering the RPC functions that will be accessible
	// once the client has authenticated
	// We pass in the ble server so that we can close the connection once the updates are installed
	// (THIS SHOULD HAPPEN OVER WIFI INSTEAD!)
	rpc_router := GetSetupRPCRouter(conn, wifi_manager, srv, pairing_ui)

	StartHTTPServer(conn, wifi_manager, pairing_ui)

	auth_handler := new(OneTimeAuthHandler)
	auth_handler.Init("spheramid")

	controlChecker = NewControlChecker(pairing_ui)

	RegisterSecuredRPCService(srv, rpc_router, auth_handler, pairing_ui)

	// Start the server
	//log.Println("Starting setup assistant...");
	//log.Fatal(srv.AdvertiseAndServe())

	states := wifi_manager.WatchState()

	//wifi_manager.WifiConfigured()

	var wireless_stale *time.Timer

	is_serving_pairer := false

	// start by forcing the state to Disconnected.
	// reloading the configuration in wpa_supplicant will also force this,
	// but we need to do it here in case we are already disconnected
	states <- WifiStateDisconnected
	wifi_manager.Controller.ReloadConfiguration()

	handleBadWireless := func() {
		logger.Warningf("Wireless is stale! Invalid SSID, router down, or not in range.")

		if !is_serving_pairer {
			is_serving_pairer = true
			logger.Infof("Launching BLE pairing assistant...")
			go func() {
				err := srv.AdvertiseAndServe()
				if err != nil {
					logger.Fatalf("failure to advertise and serve: %v", err)
				}
			}()
			// and if the hostap isn't normally active, make it active
			if !config.Wireless_Host.Always_Active {
				logger.Infof("Launching AdHoc pairing assistant...")
				apManager.StartHostAP()
			}
		}
	}

	wifi_configured, _ := wifi_manager.WifiConfigured()
	if !wifi_configured {
		// when wireless isn't configured at all, automatically start doing this, don't wait for staleness
		handleBadWireless()
	}

	if config.Wireless_Host.Enables_Control {
		// the wireless AP causes control to be enabled, so we just start the heartbeat immediately
		controlChecker.StartHeartbeat()
	}

	for {
		state := <-states
		logger.Infof("State: %v", state)

		switch state {
		case WifiStateConnected:
			if wireless_stale != nil {
				wireless_stale.Stop()
			}
			wireless_stale = nil
			logger.Infof("Connected and attempting to get IP.")

			/*if !config.Wireless_Host.Enables_Control {
				// if the wireless AP mode hasn't already enabled normal control, then enable it now that wifi works
				controlChecker.StartHeartbeat()
			}*/

			if is_serving_pairer {
				is_serving_pairer = false

				// We need to keep the server open for now, as we are sending update progress to it, and accepting
				// led drawing messages. Later, this will be over wifi and we can close it here.
				//srv.Close()

				// and if the hostap isn't normally active, turn it off again
				if !config.Wireless_Host.Always_Active {
					logger.Infof("Terminating AdHoc pairing assistant.")
					apManager.StopHostAP()
				}
			}

			if factoryReset {
				// if the app provided credentials, then we need to wait for app to acknowledge the connected state
				// before we exit otherwise it may not receive the response. So, we configure a callback which
				// will be called once the app as indicated that it has seen the connected state.
				wifi_manager.OnAcknowledgment(func() {
					logger.Infof("Network connected. Quitting.")
					os.Exit(0)
				})
			}

		case WifiStateDisconnected:
			if wireless_stale == nil {
				wireless_stale = time.AfterFunc(WirelessStaleTimeout, handleBadWireless)
			}

		case WifiStateInvalidKey:
			if wireless_stale == nil {
				wireless_stale = time.AfterFunc(WirelessStaleTimeout, handleBadWireless)
			}
			wifi_configured, _ = wifi_manager.WifiConfigured()
			if wifi_configured {
				// not stale, we actually know the key is wrong
				// FIXME: report back to the user! for now we're just going to let staleness timeout
				/*if wireless_stale != nil {
					wireless_stale.Stop()
				}
				wireless_stale = nil*/

				logger.Warningf("Wireless key is invalid!")
			}
		}
	}
}
<commit_msg>Show phone-fade icon when waiting for a phone.<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	nlog "github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/support"
	"github.com/ninjasphere/sphere-go-led-controller/model"
	"github.com/ninjasphere/sphere-go-led-controller/util"
	"github.com/paypal/gatt"
)

const WirelessNetworkInterface = "wlan0"

// consider the wifi to be invalid after this timeout
const WirelessStaleTimeout = time.Second * 30 // FIXME: INCREASE THIS. a few minutes at least when not in testing.

var firewallHook = flag.Bool("firewall-hook", false, "Sets up the firewall based on configuration options, and nothing else.")
var factoryReset = false
var imagesDir = ""

// factoryReset - we can't use anything that requires MQTT in this mode

var logger = nlog.GetLogger("sphere-setup")

func main() {
	// ap0 adhoc/hostap management
	flag.BoolVar(&factoryReset, "factory-reset", false, "Run in factory reset mode.")
	flag.StringVar(&imagesDir, "images", "/opt/ninjablocks/drivers/sphere-go-led-controller/images", "The location of the images directory.")

	go func() {
		support.WaitUntilSignal()
	}()

	config := LoadConfig("/etc/opt/ninja/setup-assistant.conf")
	apManager := NewAccessPointManager(config)

	flag.Parse()

	if *firewallHook {
		logger.Debugf("Setting ip firewall rules...")
		apManager.SetupFirewall()
		return
	}

	util.SetImageDir(imagesDir)
	var pairing_ui ConsolePairingUI
	var controlChecker *ControlChecker

	restartHeartbeat := false

	startResetMonitor(func(m *model.ResetMode) {
		if pairing_ui == nil || controlChecker == nil {
			return
		}
		if m.Mode == "none" {
			if restartHeartbeat {
				// only restart the heartbeat if we stopped it previously
				controlChecker.StartHeartbeat()
			}
		} else {
			restartHeartbeat = controlChecker.StopHeartbeat()
			pairing_ui.DisplayResetMode(m)
		}
	})

	apManager.WriteAPConfig()
	if config.Wireless_Host.Always_Active {
		apManager.StartHostAP()
	} else {
		apManager.StopHostAP()
	}

	// wlan0 client management
	wifi_manager, err := NewWifiManager(WirelessNetworkInterface)
	if err != nil {
		log.Fatal("Could not setup manager for wlan0, does the interface exist?: %v", err)
	}
	defer wifi_manager.Cleanup()

	// When in reset mode, this will talk to the led matrix directly,
	// otherwise, it will use sphere-go-led-controller via mqtt.
	pairing_ui, err = NewPairingUI()
	if err != nil {
		log.Fatal("Could not setup ninja connection")
	}

	// This name is sent in the BLE advertising packet,
	// and is used by the phone to see that this is a
	// sphere, and if it is in factory reset mode.
	serviceName := "ninjasphere"
	if factoryReset {
		serviceName = "resetsphere"
	}

	srv := &gatt.Server{
		Name: serviceName,
		Connect: func(c gatt.Conn) {
			logger.Infof("BLE Connect")
			pairing_ui.DisplayIcon("ble-connected.gif")
		},
		Disconnect: func(c gatt.Conn) {
			logger.Infof("BLE Disconnect")
			pairing_ui.DisplayIcon("ble-disconnected.gif")
		},
		StateChange: func(state string) {
			logger.Infof("BLE State Change: %s", state)
		},
	}

	var conn *ninja.Connection

	if !factoryReset {

		conn, err = ninja.Connect("sphere-setup-assistant-updates")

		if err != nil {
			logger.FatalErrorf(err, "Failed to connect to mqtt")
		}
	}

	// start by registering the RPC functions that will be accessible
	// once the client has authenticated
	// We pass in the ble server so that we can close the connection once the updates are installed
	// (THIS SHOULD HAPPEN OVER WIFI INSTEAD!)
	rpc_router := GetSetupRPCRouter(conn, wifi_manager, srv, pairing_ui)

	StartHTTPServer(conn, wifi_manager, pairing_ui)

	auth_handler := new(OneTimeAuthHandler)
	auth_handler.Init("spheramid")

	controlChecker = NewControlChecker(pairing_ui)

	RegisterSecuredRPCService(srv, rpc_router, auth_handler, pairing_ui)

	// Start the server
	//log.Println("Starting setup assistant...");
	//log.Fatal(srv.AdvertiseAndServe())

	states := wifi_manager.WatchState()

	//wifi_manager.WifiConfigured()

	var wireless_stale *time.Timer

	is_serving_pairer := false

	// start by forcing the state to Disconnected.
	// reloading the configuration in wpa_supplicant will also force this,
	// but we need to do it here in case we are already disconnected
	states <- WifiStateDisconnected
	wifi_manager.Controller.ReloadConfiguration()

	handleBadWireless := func() {
		logger.Warningf("Wireless is stale! Invalid SSID, router down, or not in range.")

		if !is_serving_pairer {
			is_serving_pairer = true

			pairing_ui.DisplayIcon("phone-fade.gif")

			logger.Infof("Launching BLE pairing assistant...")
			go func() {
				err := srv.AdvertiseAndServe()
				if err != nil {
					logger.Fatalf("failure to advertise and serve: %v", err)
				}
			}()
			// and if the hostap isn't normally active, make it active
			if !config.Wireless_Host.Always_Active {
				logger.Infof("Launching AdHoc pairing assistant...")
				apManager.StartHostAP()
			}
		}
	}

	wifi_configured, _ := wifi_manager.WifiConfigured()
	if !wifi_configured {
		// when wireless isn't configured at all, automatically start doing this, don't wait for staleness
		handleBadWireless()
	}

	if config.Wireless_Host.Enables_Control {
		// the wireless AP causes control to be enabled, so we just start the heartbeat immediately
		controlChecker.StartHeartbeat()
	}

	for {
		state := <-states
		logger.Infof("State: %v", state)

		switch state {
		case WifiStateConnected:
			if wireless_stale != nil {
				wireless_stale.Stop()
			}
			wireless_stale = nil
			logger.Infof("Connected and attempting to get IP.")

			/*if !config.Wireless_Host.Enables_Control {
				// if the wireless AP mode hasn't already enabled normal control, then enable it now that wifi works
				controlChecker.StartHeartbeat()
			}*/

			if is_serving_pairer {
				is_serving_pairer = false

				// We need to keep the server open for now, as we are sending update progress to it, and accepting
				// led drawing messages. Later, this will be over wifi and we can close it here.
				//srv.Close()

				// and if the hostap isn't normally active, turn it off again
				if !config.Wireless_Host.Always_Active {
					logger.Infof("Terminating AdHoc pairing assistant.")
					apManager.StopHostAP()
				}
			}

			if factoryReset {
				// if the app provided credentials, then we need to wait for app to acknowledge the connected state
				// before we exit otherwise it may not receive the response. So, we configure a callback which
				// will be called once the app as indicated that it has seen the connected state.
				wifi_manager.OnAcknowledgment(func() {
					logger.Infof("Network connected. Quitting.")
					os.Exit(0)
				})
			}

		case WifiStateDisconnected:
			if wireless_stale == nil {
				wireless_stale = time.AfterFunc(WirelessStaleTimeout, handleBadWireless)
			}

		case WifiStateInvalidKey:
			if wireless_stale == nil {
				wireless_stale = time.AfterFunc(WirelessStaleTimeout, handleBadWireless)
			}
			wifi_configured, _ = wifi_manager.WifiConfigured()
			if wifi_configured {
				// not stale, we actually know the key is wrong
				// FIXME: report back to the user! for now we're just going to let staleness timeout
				/*if wireless_stale != nil {
					wireless_stale.Stop()
				}
				wireless_stale = nil*/

				logger.Warningf("Wireless key is invalid!")
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/vad/elasticsearch_exporter/parser"
)

var (
	es           = flag.String("es", "http://localhost:9200", "ES URL")
	bind         = flag.String("bind", ":9092", "Address to bind to")
	timeInterval = flag.Int("time", 5, "Time interval between scrape runs, in seconds")

	up = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "es_up",
		Help: "Current status of ES",
	})

	metrics []*parser.Metric
)

func init() {
	metrics = []*parser.Metric{
		gcPoolCount("young"),
		gcPoolCount("old"),
		gcPoolTime("young"),
		gcPoolTime("old"),
		memPool("young", "used"),
		memPool("old", "used"),
		memPool("young", "max"),
		memPool("old", "max"),
		heap("max"),
		heap("used"),
		raw("indices.merges.total"),
		raw("indices.merges.total_time_in_millis"),
		raw("indices.merges.total_docs"),
		raw("indices.merges.total_size_in_bytes"),
		raw("indices.merges.total_throttled_time_in_millis"),
		raw("indices.warmer.total"),
		raw("indices.warmer.total_time_in_millis"),
		raw("indices.fielddata.memory_size_in_bytes"),
		raw("indices.segments.count"),
		raw("indices.segments.memory_in_bytes"),
		raw("indices.segments.terms_memory_in_bytes"),
		raw("indices.segments.stored_fields_memory_in_bytes"),
		raw("indices.segments.term_vectors_memory_in_bytes"),
		raw("indices.segments.norms_memory_in_bytes"),
		raw("indices.segments.points_memory_in_bytes"),
		raw("indices.segments.doc_values_memory_in_bytes"),
		raw("indices.segments.index_writer_memory_in_bytes"),
		raw("indices.segments.version_map_memory_in_bytes"),
		raw("indices.request_cache.memory_size_in_bytes"),
		raw("indices.request_cache.evictions"),
		raw("indices.request_cache.hit_count"),
		raw("indices.request_cache.miss_count"),
		raw("indices.docs.count"),
		raw("indices.docs.deleted"),
	}
	addToMetrics(totalAndMillis("indices.search.query"))
	addToMetrics(totalAndMillis("indices.search.scroll"))
	addToMetrics(totalAndMillis("indices.indexing.index"))
	addToMetrics(totalAndMillis("indices.indexing.delete"))

	prometheus.MustRegister(up)
	for _, metric := range metrics {
		prometheus.MustRegister(metric.Gauge)
	}
}

func heap(t string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_memory_heap_%s_bytes", t),
		fmt.Sprintf("%s heap in bytes", t),
		fmt.Sprintf("jvm.mem.heap_%s_in_bytes", t),
	)
}

func memPool(pool, t string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_memory_pool_%s_%s_bytes", pool, t),
		fmt.Sprintf("%s memory of pool %s", t, pool),
		fmt.Sprintf("jvm.mem.pools.%s.%s_in_bytes", pool, t),
	)
}

func gcPoolTime(pool string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_gc_%s_collection_time_ms", pool),
		fmt.Sprintf("Time of collections of %s GC", pool),
		fmt.Sprintf("jvm.gc.collectors.%s.collection_time_in_millis", pool),
	)
}

func gcPoolCount(pool string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_gc_%s_collection_count", pool),
		fmt.Sprintf("Number of collections of %s GC", pool),
		fmt.Sprintf("jvm.gc.collectors.%s.collection_count", pool),
	)
}

func raw(op string) *parser.Metric {
	o := strings.Replace(op, ".", "_", -1)
	return parser.NewMetric(fmt.Sprintf("es_%s", o), op, op)
}

func totalAndMillis(m string) []*parser.Metric {
	var out []*parser.Metric

	out = make([]*parser.Metric, 2)
	out[0] = raw(m + "_total")
	out[1] = raw(m + "_time_in_millis")
	return out
}

func addToMetrics(m []*parser.Metric) {
	metrics = append(metrics, m...)
}

func scrape(ns string) {
	resp, err := http.Get(ns)

	if err != nil {
		up.Set(0)
		log.Println(err.Error())
		return
	}
	defer resp.Body.Close()
	v, err := parser.NewNodeStatsJson(resp.Body)

	if err != nil {
		up.Set(0)
		log.Println("Error decoding ES JSON:", err.Error())
		return
	}

	for nodeName, jobject := range v.Nodes {
		for _, metric := range metrics {
			err := metric.Observe(nodeName, jobject)
			if err != nil {
				log.Println("Error observing metric from '", metric.Path, "' ", err.Error())
			}
		}
	}

	up.Set(1)
}

func scrapeForever() {
	ns := strings.TrimRight(*es, "/") + "/_nodes/stats"
	for {
		scrape(ns)

		time.Sleep(time.Duration(*timeInterval) * time.Second)
	}
}

func main() {
	flag.Parse()
	if *timeInterval < 1 {
		log.Fatal("Time interval must be >= 1")
	}

	go scrapeForever()

	http.Handle("/metrics", promhttp.Handler())

	log.Println("Listen on address", *bind)
	log.Fatal(http.ListenAndServe(*bind, nil))
}
<commit_msg>add more metrics<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/vad/elasticsearch_exporter/parser"
)

var (
	es           = flag.String("es", "http://localhost:9200", "ES URL")
	bind         = flag.String("bind", ":9092", "Address to bind to")
	timeInterval = flag.Int("time", 5, "Time interval between scrape runs, in seconds")

	up = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "es_up",
		Help: "Current status of ES",
	})

	metrics []*parser.Metric
)

func init() {
	metrics = []*parser.Metric{
		gcPoolCount("young"),
		gcPoolCount("old"),
		gcPoolTime("young"),
		gcPoolTime("old"),
		memPool("young", "used"),
		memPool("old", "used"),
		memPool("young", "max"),
		memPool("old", "max"),
		heap("max"),
		heap("used"),
		raw("indices.merges.total"),
		raw("indices.merges.total_time_in_millis"),
		raw("indices.merges.total_docs"),
		raw("indices.merges.total_size_in_bytes"),
		raw("indices.merges.total_throttled_time_in_millis"),
		raw("indices.warmer.total"),
		raw("indices.warmer.total_time_in_millis"),
		raw("indices.fielddata.memory_size_in_bytes"),
		raw("indices.segments.count"),
		raw("indices.segments.memory_in_bytes"),
		raw("indices.segments.terms_memory_in_bytes"),
		raw("indices.segments.stored_fields_memory_in_bytes"),
		raw("indices.segments.term_vectors_memory_in_bytes"),
		raw("indices.segments.norms_memory_in_bytes"),
		raw("indices.segments.points_memory_in_bytes"),
		raw("indices.segments.doc_values_memory_in_bytes"),
		raw("indices.segments.index_writer_memory_in_bytes"),
		raw("indices.segments.version_map_memory_in_bytes"),
		raw("indices.request_cache.memory_size_in_bytes"),
		raw("indices.request_cache.evictions"),
		raw("indices.request_cache.hit_count"),
		raw("indices.request_cache.miss_count"),
		raw("indices.docs.count"),
		raw("indices.docs.deleted"),
		raw("indices.query_cache.memory_size_in_bytes"),
		raw("indices.query_cache.total_count"),
		raw("indices.query_cache.hit_count"),
		raw("indices.query_cache.miss_count"),
		raw("indices.query_cache.cache_size"),
		raw("indices.query_cache.cache_count"),
		raw("indices.query_cache.evictions"),
		raw("indices.recovery.throttle_time_in_millis"),
	}
	addToMetrics(totalAndMillis("indices.search.fetch"))
	addToMetrics(totalAndMillis("indices.search.query"))
	addToMetrics(totalAndMillis("indices.search.scroll"))
	addToMetrics(totalAndMillis("indices.indexing.index"))
	addToMetrics(totalAndMillis("indices.indexing.delete"))

	prometheus.MustRegister(up)
	for _, metric := range metrics {
		prometheus.MustRegister(metric.Gauge)
	}
}

func heap(t string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_memory_heap_%s_bytes", t),
		fmt.Sprintf("%s heap in bytes", t),
		fmt.Sprintf("jvm.mem.heap_%s_in_bytes", t),
	)
}

func memPool(pool, t string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_memory_pool_%s_%s_bytes", pool, t),
		fmt.Sprintf("%s memory of pool %s", t, pool),
		fmt.Sprintf("jvm.mem.pools.%s.%s_in_bytes", pool, t),
	)
}

func gcPoolTime(pool string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_gc_%s_collection_time_ms", pool),
		fmt.Sprintf("Time of collections of %s GC", pool),
		fmt.Sprintf("jvm.gc.collectors.%s.collection_time_in_millis", pool),
	)
}

func gcPoolCount(pool string) *parser.Metric {
	return parser.NewMetric(
		fmt.Sprintf("es_gc_%s_collection_count", pool),
		fmt.Sprintf("Number of collections of %s GC", pool),
		fmt.Sprintf("jvm.gc.collectors.%s.collection_count", pool),
	)
}

func raw(op string) *parser.Metric {
	o := strings.Replace(op, ".", "_", -1)
	return parser.NewMetric(fmt.Sprintf("es_%s", o), op, op)
}

func totalAndMillis(m string) []*parser.Metric {
	var out []*parser.Metric

	out = make([]*parser.Metric, 2)
	out[0] = raw(m + "_total")
	out[1] = raw(m + "_time_in_millis")
	return out
}

func addToMetrics(m []*parser.Metric) {
	metrics = append(metrics, m...)
}

func scrape(ns string) {
	resp, err := http.Get(ns)

	if err != nil {
		up.Set(0)
		log.Println(err.Error())
		return
	}
	defer resp.Body.Close()
	v, err := parser.NewNodeStatsJson(resp.Body)

	if err != nil {
		up.Set(0)
		log.Println("Error decoding ES JSON:", err.Error())
		return
	}

	for nodeName, jobject := range v.Nodes {
		for _, metric := range metrics {
			err := metric.Observe(nodeName, jobject)
			if err != nil {
				log.Println("Error observing metric from '", metric.Path, "' ", err.Error())
			}
		}
	}

	up.Set(1)
}

func scrapeForever() {
	ns := strings.TrimRight(*es, "/") + "/_nodes/stats"
	for {
		scrape(ns)

		time.Sleep(time.Duration(*timeInterval) * time.Second)
	}
}

func main() {
	flag.Parse()
	if *timeInterval < 1 {
		log.Fatal("Time interval must be >= 1")
	}

	go scrapeForever()

	http.Handle("/metrics", promhttp.Handler())

	log.Println("Listen on address", *bind)
	log.Fatal(http.ListenAndServe(*bind, nil))
}
<|endoftext|>


<commit_before>package misc

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	// "runtime/debug"
)

func Contains(list []interface{}, elem interface{}) bool {
	for _, t := range list {
		if t == elem {
			return true
		}
	}
	return false
}

func Errorf(format string, a ...interface{}) error {
	return fmt.Errorf(format, a...)
}

func Exists(file string) bool {
	// see also http://stackoverflow.com/a/10510783/1034080
	_, err := os.Stat(file)
	return err == nil // !os.IsNotExist(err)
}

func Fatal(a ...interface{}) {
	// debug.PrintStack()
	fmt.Println(a...)
	os.Exit(1)
}

func Fatalf(format string, a ...interface{}) {
	// debug.PrintStack()
	fmt.Printf(format+"\n", a...)
	os.Exit(1)
}

func FatalIf(err error) {
	if err != nil {
		Fatal(err)
	}
}

func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
	return fmt.Fprintf(w, format, a...)
}

func OpenOrCreate(name string) (file *os.File, err error) {
	f, err := os.Open(name)
	if err == nil {
		// no error, all good
		return f, nil
	}
	if !os.IsNotExist(err) {
		// err does not indicate that the file does not exist
		return nil, err
	}
	// file does not exist, try to create it
	f, err = os.Create(name)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func PathJoin(a ...string) string {
	return filepath.Join(a...)
}

func Print(a ...interface{}) (n int, err error) {
	return fmt.Print(a...)
}

func Printf(format string, a ...interface{}) (n int, err error) {
	return fmt.Printf(format, a...)
}

func Println(a ...interface{}) (n int, err error) {
	return fmt.Println(a...)
}

func Sprintf(format string, a ...interface{}) string {
	return fmt.Sprintf(format, a...)
}

func ThisDirN(n int) string {
	_, filename, _, ok := runtime.Caller(n)
	if !ok {
		Fatal("runtime.Caller() was not ok")
	}
	return filepath.Dir(filename)
}

func ThisDir() string {
	return ThisDirN(2)
}

func ThisDirJoin(a ...string) string {
	return filepath.Join(append([]string{ThisDirN(2)}, a...)...)
}
<commit_msg>Add Fprint<commit_after>package misc

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	// "runtime/debug"
)

func Contains(list []interface{}, elem interface{}) bool {
	for _, t := range list {
		if t == elem {
			return true
		}
	}
	return false
}

func Errorf(format string, a ...interface{}) error {
	return fmt.Errorf(format, a...)
}

func Exists(file string) bool {
	// see also http://stackoverflow.com/a/10510783/1034080
	_, err := os.Stat(file)
	return err == nil // !os.IsNotExist(err)
}

func Fatal(a ...interface{}) {
	// debug.PrintStack()
	fmt.Println(a...)
	os.Exit(1)
}

func Fatalf(format string, a ...interface{}) {
	// debug.PrintStack()
	fmt.Printf(format+"\n", a...)
	os.Exit(1)
}

func FatalIf(err error) {
	if err != nil {
		Fatal(err)
	}
}

func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
	return fmt.Fprint(w, a...)
}

func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) {
	return fmt.Fprintf(w, format, a...)
}

func OpenOrCreate(name string) (file *os.File, err error) {
	f, err := os.Open(name)
	if err == nil {
		// no error, all good
		return f, nil
	}
	if !os.IsNotExist(err) {
		// err does not indicate that the file does not exist
		return nil, err
	}
	// file does not exist, try to create it
	f, err = os.Create(name)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func PathJoin(a ...string) string {
	return filepath.Join(a...)
}

func Print(a ...interface{}) (n int, err error) {
	return fmt.Print(a...)
}

func Printf(format string, a ...interface{}) (n int, err error) {
	return fmt.Printf(format, a...)
}

func Println(a ...interface{}) (n int, err error) {
	return fmt.Println(a...)
}

func Sprintf(format string, a ...interface{}) string {
	return fmt.Sprintf(format, a...)
}

func ThisDirN(n int) string {
	_, filename, _, ok := runtime.Caller(n)
	if !ok {
		Fatal("runtime.Caller() was not ok")
	}
	return filepath.Dir(filename)
}

func ThisDir() string {
	return ThisDirN(2)
}

func ThisDirJoin(a ...string) string {
	return filepath.Join(append([]string{ThisDirN(2)}, a...)...)
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package toybroker

import (
	"fmt"
	"net"
)

func MqttMainLoop(conn net.Conn, topics Topics, hook Hook) {
	connect, _, _, _, _, remaining, err := readMessage(conn)
	if connect != CONNECT || err != nil {
		conn.Write([]byte{CONNACK * 16, 2, 0, CONNACK_Rejected})
		return
	}
	clientID, _, _, loginName, loginPassword, err := unpackCONNECT(remaining)
	status := hook.Login(conn, clientID, loginName, loginPassword)
	if status != CONNACK_Success {
		conn.Write([]byte{CONNACK * 16, 2, 0, CONNACK_Rejected})
		return
	}

	oldClient := hook.GetClient(clientID)
	client := NewClient(clientID, loginName, conn, oldClient)
	hook.SetClient(client)
	client.Send(packCONNACK(status))

	for {
		command, _, header_qos, retain, _, remaining, err := readMessage(conn)
		if err != nil {
			hook.Logout(clientID)
			break
		}
		switch command {
		case PUBLISH:
			topic, messageID, payload, err := unpackPUBLISH(remaining)
			debugOutput(fmt.Sprintf("PUBLISH:%s,%d,%v,%v", topic, messageID, payload, err))

			clientList, qosList := topics.List(topic)
			for i, clientID := range clientList {
				target := hook.GetClient(clientID)
				if qosList[i] > 0 && header_qos > 0 {
					qosList[i] = 1
				}
				target.Publish(false, qosList[i], topic, payload)

				if retain {
					debugOutput("AddRetainMessage()")
					topics.AddRetainMessage(topic, payload)
				} else {
					debugOutput("Not retain")
				}
			}
		case PUBACK:
			debugOutput("PUBACK")
		case PUBREL:
			debugOutput("PUBREL")
		case SUBSCRIBE:
			messageID, subscribe_topics, err := unpackSUBSCRIBE(remaining)
			debugOutput(fmt.Sprintf("SUBSCRIBE:%v,%d,%v", subscribe_topics, messageID, err))
			qos := make([]byte, len(subscribe_topics))
			for i, topicName := range subscribe_topics {
				qos[i] = byte(hook.Subscribe(topics, topicName, clientID, 0))
			}
			client.Send(packSUBACK(messageID, qos))

			// Publish retain messages
			for i, topicName := range subscribe_topics {
				payload := topics.GetRetainMessage(topicName)
				if len(payload) != 0 {
					debugOutput(fmt.Sprintf("SendRetainMessage:%s->%v", topicName, payload))
					client.Publish(false, int(qos[i]), topicName, payload)
				}
			}

		case UNSUBSCRIBE:
			messageID, unsubscribe_topics, err := unpackUNSUBSCRIBE(remaining)
			debugOutput(fmt.Sprintf("UNSUBSCRIBE:%v,%d,%v", unsubscribe_topics, messageID, err))
			for _, topicName := range unsubscribe_topics {
				hook.Unsubscribe(topics, topicName, clientID)
			}
			client.Send(packUNSUBACK(messageID))
		case PINGREQ:
			debugOutput("PINGREQ")
			client.Send(packPINGRESP())
		case DISCONNECT:
			debugOutput("DISCONNECT")
			hook.Logout(clientID)
			break
		default:
			debugOutput("Invalid Command")
			hook.Logout(clientID)
			break
		}
	}
}
<commit_msg>add outputDebug()<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2015 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*******************************************************************************/

package toybroker

import (
	"fmt"
	"net"
)

func MqttMainLoop(conn net.Conn, topics Topics, hook Hook) {
	connect, _, _, _, _, remaining, err := readMessage(conn)
	if connect != CONNECT || err != nil {
		conn.Write([]byte{CONNACK * 16, 2, 0, CONNACK_Rejected})
		return
	}
	clientID, _, _, loginName, loginPassword, err := unpackCONNECT(remaining)
	status := hook.Login(conn, clientID, loginName, loginPassword)
	if status != CONNACK_Success {
		conn.Write([]byte{CONNACK * 16, 2, 0, CONNACK_Rejected})
		return
	}

	oldClient := hook.GetClient(clientID)
	client := NewClient(clientID, loginName, conn, oldClient)
	hook.SetClient(client)
	client.Send(packCONNACK(status))

	for {
		command, _, header_qos, retain, _, remaining, err := readMessage(conn)
		if err != nil {
			hook.Logout(clientID)
			break
		}
		switch command {
		case PUBLISH:
			topic, messageID, payload, err := unpackPUBLISH(remaining)
			debugOutput(fmt.Sprintf("PUBLISH:%s,%d,%v,%v", topic, messageID, payload, err))

			clientList, qosList := topics.List(topic)
			for i, clientID := range clientList {
				target := hook.GetClient(clientID)
				if qosList[i] > 0 && header_qos > 0 {
					qosList[i] = 1
				}
				target.Publish(false, qosList[i], topic, payload)

				if retain {
					debugOutput(fmt.Sprintf("AddRetainMessage(%s)", topic))
					topics.AddRetainMessage(topic, payload)
				} else {
					debugOutput("Not retain")
				}
			}
		case PUBACK:
			debugOutput("PUBACK")
		case PUBREL:
			debugOutput("PUBREL")
		case SUBSCRIBE:
			messageID, subscribe_topics, err := unpackSUBSCRIBE(remaining)
			debugOutput(fmt.Sprintf("SUBSCRIBE:%v,%d,%v", subscribe_topics, messageID, err))
			qos := make([]byte, len(subscribe_topics))
			for i, topicName := range subscribe_topics {
				qos[i] = byte(hook.Subscribe(topics, topicName, clientID, 0))
			}
			client.Send(packSUBACK(messageID, qos))

			// Publish retain messages
			for i, topicName := range subscribe_topics {
				payload := topics.GetRetainMessage(topicName)
				debugOutput(fmt.Sprintf("GetRetainMessage(%s)=%v", topicName, payload))

				if len(payload) != 0 {
					debugOutput(fmt.Sprintf("SendRetainMessage:%s->%v", topicName, payload))
					client.Publish(false, int(qos[i]), topicName, payload)
				}
			}

		case UNSUBSCRIBE:
			messageID, unsubscribe_topics, err := unpackUNSUBSCRIBE(remaining)
			debugOutput(fmt.Sprintf("UNSUBSCRIBE:%v,%d,%v", unsubscribe_topics, messageID, err))
			for _, topicName := range unsubscribe_topics {
				hook.Unsubscribe(topics, topicName, clientID)
			}
			client.Send(packUNSUBACK(messageID))
		case PINGREQ:
			debugOutput("PINGREQ")
			client.Send(packPINGRESP())
		case DISCONNECT:
			debugOutput("DISCONNECT")
			hook.Logout(clientID)
			break
		default:
			debugOutput("Invalid Command")
			hook.Logout(clientID)
			break
		}
	}
}
<|endoftext|>


<commit_before>package GoSDK

import (
	"crypto/tls"
	"errors"
	mqtt "github.com/clearblade/mqtt_parsing"
	mqcli "github.com/clearblade/mqttclient"
)

const (
	QOS_AtMostOnce = iota
	QOS_AtLeastOnce
	QOS_PreciselyOnce
)

//InitializeMqttClient allocates a mqtt client.
//the values for initialization are drawn from the client struct
//with the exception of the timeout and client id, which is mqtt specific.
func (c *Client) InitializeMqttClient(clientid string, timeout int) error {
	var tok string
	if ut := c.GetUserToken(); ut != "" {
		tok = ut
	} else if dt := c.GetDevToken(); dt != "" {
		tok = dt
	} else if c.SystemKey == "" && tok == "" {
		//if there is no token, we are presently unable to log in
		//there is a method of pure mqtt authentication in development
		//but presently we require a token to be acquired via http
		return errors.New("System Key nor Token not present, cannot initialize mqtt")
		//The client is presently (2014-10-31) allowed
		//to try to initialize with purely a system key and secret.
		//That is discontinued legacy behavior
		//so don't rely on it!
	}
	c.MQTTClient = mqcli.NewClient(tok,
		c.SystemKey,
		c.SystemSecret,
		clientid,
		timeout)
	return nil
}

//Below are a series of convience functions to allow the user to only need to import
//the clearblade go-sdk

//ConnectToBroker connects to the broker and sends the connect packet
func (c *Client) ConnectToBroker(address string, ssl *tls.Config) error {
	err := c.MQTTClient.Start(address, ssl)
	if err != nil {
		return err
	}
	return mqcli.SendConnect(c.MQTTClient)
}

//PublishString is a simple helper to create a publish with a string payload
func (c *Client) PublishString(topic, data string, qos int) error {
	///TODO: threadsafe random
	pub := mqcli.MakeMeAPublish(topic, data, uint16(c.mrand.Int()))
	return mqcli.PublishFlow(c.MQTTClient, pub)
}

func (c *Client) PublishBytes(topic string, data []byte, qos int) error {
	//TODO:threadsafe random
	pub := mqcli.MakeMeABytePublish(topic, data, uint16(c.mrand.Int()))
	return mqcli.PublishFlow(c.MQTTClient, pub)
}

//Subscribe is a simple wrapper around the mqtt client library
func (c *Client) Subscribe(topic string, qos int) (<-chan mqtt.Message, error) {
	return mqcli.SubscribeFlow(c.MQTTClient, topic, qos)
}

//Unsubscribe is a simple wrapper around the mqtt client library
func (c *Client) Unsubscribe(topic string) error {
	return mqcli.UnsubscribeFlow(c.MQTTClient, topic)
}

//Disconnect is a simple wrapper for sending mqtt disconnects
func (c *Client) Disconnect() error {
	return mqcli.SendDisconnect(c.MQTTClient)
}
<commit_msg>added nil checks as discussed<commit_after>package GoSDK

import (
	"crypto/tls"
	"errors"
	mqtt "github.com/clearblade/mqtt_parsing"
	mqcli "github.com/clearblade/mqttclient"
)

const (
	QOS_AtMostOnce = iota
	QOS_AtLeastOnce
	QOS_PreciselyOnce
)

//InitializeMqttClient allocates a mqtt client.
//the values for initialization are drawn from the client struct
//with the exception of the timeout and client id, which is mqtt specific.
func (c *Client) InitializeMqttClient(clientid string, timeout int) error {
	var tok string
	if ut := c.GetUserToken(); ut != "" {
		tok = ut
	} else if dt := c.GetDevToken(); dt != "" {
		tok = dt
	} else if c.SystemKey == "" && tok == "" {
		//if there is no token, we are presently unable to log in
		//there is a method of pure mqtt authentication in development
		//but presently we require a token to be acquired via http
		return errors.New("System Key nor Token not present, cannot initialize mqtt")
		//The client is presently (2014-10-31) allowed
		//to try to initialize with purely a system key and secret.
		//That is discontinued legacy behavior
		//so don't rely on it!
	}
	c.MQTTClient = mqcli.NewClient(tok,
		c.SystemKey,
		c.SystemSecret,
		clientid,
		timeout)
	return nil
}

//Below are a series of convience functions to allow the user to only need to import
//the clearblade go-sdk

//ConnectToBroker connects to the broker and sends the connect packet
func (c *Client) ConnectToBroker(address string, ssl *tls.Config) error {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	err := c.MQTTClient.Start(address, ssl)
	if err != nil {
		return err
	}
	return mqcli.SendConnect(c.MQTTClient)
}

//PublishString is a simple helper to create a publish with a string payload
func (c *Client) PublishString(topic, data string, qos int) error {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	pub := mqcli.MakeMeAPublish(topic, data, uint16(c.mrand.Int()))
	return mqcli.PublishFlow(c.MQTTClient, pub)
}

func (c *Client) PublishBytes(topic string, data []byte, qos int) error {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	pub := mqcli.MakeMeABytePublish(topic, data, uint16(c.mrand.Int()))
	return mqcli.PublishFlow(c.MQTTClient, pub)
}

//Subscribe is a simple wrapper around the mqtt client library
func (c *Client) Subscribe(topic string, qos int) (<-chan mqtt.Message, error) {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	return mqcli.SubscribeFlow(c.MQTTClient, topic, qos)
}

//Unsubscribe is a simple wrapper around the mqtt client library
func (c *Client) Unsubscribe(topic string) error {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	return mqcli.UnsubscribeFlow(c.MQTTClient, topic)
}

//Disconnect is a simple wrapper for sending mqtt disconnects
func (c *Client) Disconnect() error {
	if c.MQTTClient == nil {
		return errors.New("MQTTClient is uninitialized")
	}
	return mqcli.SendDisconnect(c.MQTTClient)
}
<|endoftext|>


<commit_before>package berlingo

// Node represents a single node on the Map
type Node struct {
	Map  *Map
	Id   int
	Type *NodeType

	Paths              map[int]*Node
	Paths_Outbound     map[int]*Node
	Paths_Inbound      map[int]*Node
	Player_Id          int
	Number_Of_Soldiers int
	Incoming_Soldiers  int
	Available_Soldiers int
}

func NewNode(m *Map) *Node {
	return &Node{
		Map:            m,
		Paths:          make(map[int]*Node),
		Paths_Outbound: make(map[int]*Node),
		Paths_Inbound:  make(map[int]*Node),
	}
}

// Sets up a unidirectional link pointing from this node towards another
func (node *Node) link_to(other *Node) {

	node.Paths_Outbound[other.Id] = other
	node.Paths[other.Id] = other

	other.Paths_Inbound[node.Id] = node
	other.Paths[node.Id] = node
}

func (node *Node) reset() {
	node.Incoming_Soldiers = 0
	node.Available_Soldiers = 0
	if node.IsOwned() {
		node.Available_Soldiers = node.Number_Of_Soldiers
	}
}

// IsFree returns whether the node is free, or owned by any player
func (node *Node) IsFree() bool {
	return node.Player_Id < 0
}

// IsOwned returns whether this node is owned by the current player
//
// Note - this deviates from the ruby client implementation, where ruby's owned? is essentially the opposite of free? - this is quite confusing as naturally asking a node.IsOwned() most likely indicates the caller wants to know if they own it themselves
//
// Callers who wish to mimick the owned? behavior of the ruby client may simply ask for !node.IsFree()
func (node *Node) IsOwned() bool {
	return node.IsOwnedBy(node.Map.Game.Player_Id)
}

func (node *Node) IsOwnedBy(player_id int) bool {
	return node.Player_Id == player_id
}

func (node *Node) IsEnemy() bool {
	return !node.IsFree() && !node.IsOwned()
}

// Returns one of "free", "enemy", "owned"
func (node *Node) OwnershipStatus() string {
	if node.IsFree() {
		return "free"
	} else if node.IsEnemy() {
		return "enemy"
	} else if node.IsOwned() {
		return "owned"
	} else {
		panic("Illegal ownership status calculation")
	}
}

func (node *Node) IsControlled() bool {
	return node.IsOwned() && node.Number_Of_Soldiers > 0
}

func (node *Node) HasOutboundPathTo(other_node *Node) bool {
	_, ok := node.Paths_Outbound[other_node.Id]
	return ok
}

func (node *Node) HasInboundPathTo(other_node *Node) bool {
	_, ok := node.Paths_Inbound[other_node.Id]
	return ok
}

func (node *Node) IsAdjacentTo(other_node *Node) bool {
	_, ok := node.Paths[other_node.Id]
	return ok
}

func (node *Node) AdjacentNodes() (nodes []*Node) {
	nodes = make([]*Node, 0, len(node.Paths))
	for _, node := range node.Paths {
		nodes = append(nodes, node)
	}
	return nodes
}
<commit_msg>Rename Node.HasInboundPathTo() to Node.HasInboundPathFrom()<commit_after>package berlingo

// Node represents a single node on the Map
type Node struct {
	Map  *Map
	Id   int
	Type *NodeType

	Paths              map[int]*Node
	Paths_Outbound     map[int]*Node
	Paths_Inbound      map[int]*Node
	Player_Id          int
	Number_Of_Soldiers int
	Incoming_Soldiers  int
	Available_Soldiers int
}

func NewNode(m *Map) *Node {
	return &Node{
		Map:            m,
		Paths:          make(map[int]*Node),
		Paths_Outbound: make(map[int]*Node),
		Paths_Inbound:  make(map[int]*Node),
	}
}

// Sets up a unidirectional link pointing from this node towards another
func (node *Node) link_to(other *Node) {

	node.Paths_Outbound[other.Id] = other
	node.Paths[other.Id] = other

	other.Paths_Inbound[node.Id] = node
	other.Paths[node.Id] = node
}

func (node *Node) reset() {
	node.Incoming_Soldiers = 0
	node.Available_Soldiers = 0
	if node.IsOwned() {
		node.Available_Soldiers = node.Number_Of_Soldiers
	}
}

// IsFree returns whether the node is free, or owned by any player
func (node *Node) IsFree() bool {
	return node.Player_Id < 0
}

// IsOwned returns whether this node is owned by the current player
//
// Note - this deviates from the ruby client implementation, where ruby's owned? is essentially the opposite of free? - this is quite confusing as naturally asking a node.IsOwned() most likely indicates the caller wants to know if they own it themselves
//
// Callers who wish to mimick the owned? behavior of the ruby client may simply ask for !node.IsFree()
func (node *Node) IsOwned() bool {
	return node.IsOwnedBy(node.Map.Game.Player_Id)
}

func (node *Node) IsOwnedBy(player_id int) bool {
	return node.Player_Id == player_id
}

func (node *Node) IsEnemy() bool {
	return !node.IsFree() && !node.IsOwned()
}

// Returns one of "free", "enemy", "owned"
func (node *Node) OwnershipStatus() string {
	if node.IsFree() {
		return "free"
	} else if node.IsEnemy() {
		return "enemy"
	} else if node.IsOwned() {
		return "owned"
	} else {
		panic("Illegal ownership status calculation")
	}
}

func (node *Node) IsControlled() bool {
	return node.IsOwned() && node.Number_Of_Soldiers > 0
}

func (node *Node) HasOutboundPathTo(other_node *Node) bool {
	_, ok := node.Paths_Outbound[other_node.Id]
	return ok
}

func (node *Node) HasInboundPathFrom(other_node *Node) bool {
	_, ok := node.Paths_Inbound[other_node.Id]
	return ok
}

func (node *Node) IsAdjacentTo(other_node *Node) bool {
	_, ok := node.Paths[other_node.Id]
	return ok
}

func (node *Node) AdjacentNodes() (nodes []*Node) {
	nodes = make([]*Node, 0, len(node.Paths))
	for _, node := range node.Paths {
		nodes = append(nodes, node)
	}
	return nodes
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
)

// Node type
type Node struct {
	Path string // File path
	Size int64  // File size
	Hash string // String form of SHA1 hash
}

func (n *Node) Value() interface{} {
	return n
}

const blockSize = 1024 // Guestimate of a FS block-size for optimal read call

// Calculate hash
func (node *Node) calculateHash(fast bool) string {
	// Open file
	file, err := os.Open(node.Path)
	if err != nil {
		log.Println("WARN", err)
		return ""
	}
	defer file.Close()

	var n int64 // bytes read
	hash := sha1.New()

	if fast && node.Size > blockSize {
		// Make one read call
		_, err = io.CopyN(hash, file, blockSize)

	} else {
		// Filesystem and memory optimal read
		n, err = io.Copy(hash, file)

		// Paranoid sanity check
		if n != node.Size {
			err = errors.New("Partial read")
			log.Println("WARN", err)
			return ""
		}
	}

	// Check for normal errors
	if err != nil {
		log.Println("WARN", err)
		return ""
	}

	// Add hash value
	return fmt.Sprintf("%0x", hash.Sum(nil))
}
<commit_msg>- Added file name to the "partial read" warning<commit_after>package main

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
)

// Node type
type Node struct {
	Path string // File path
	Size int64  // File size
	Hash string // String form of SHA1 hash
}

func (n *Node) Value() interface{} {
	return n
}

const blockSize = 1024 // Guestimate of a FS block-size for optimal read call

// Calculate hash
func (node *Node) calculateHash(fast bool) string {
	// Open file
	file, err := os.Open(node.Path)
	if err != nil {
		log.Println("WARN", err)
		return ""
	}
	defer file.Close()

	var n int64 // bytes read
	hash := sha1.New()

	if fast && node.Size > blockSize {
		// Make one read call
		_, err = io.CopyN(hash, file, blockSize)

	} else {
		// Filesystem and memory optimal read
		n, err = io.Copy(hash, file)

		// Paranoid sanity check
		if n != node.Size {
			err = errors.New("Partial read: " + node.Path)
			log.Println("WARN", err)
			return ""
		}
	}

	// Check for normal errors
	if err != nil {
		log.Println("WARN", err)
		return ""
	}

	// Add hash value
	return fmt.Sprintf("%0x", hash.Sum(nil))
}
<|endoftext|>