<commit_before>package main

import (
	"bufio"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/go-martini/martini"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/kardianos/osext"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/natefinch/lumberjack"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk"
	"github.com/QubitProducts/bamboo/api"
	"github.com/QubitProducts/bamboo/configuration"
	"github.com/QubitProducts/bamboo/qzk"
	"github.com/QubitProducts/bamboo/services/event_bus"
)

/*
	Commandline arguments
*/
var configFilePath string
var logPath string
var serverBindPort string

func init() {
	flag.StringVar(&configFilePath, "config", "config/development.json", "Full path of the configuration JSON file")
	flag.StringVar(&logPath, "log", "", "Log path to a file. Default logs to stdout")
	flag.StringVar(&serverBindPort, "bind", ":8000", "Bind HTTP server to a specific port")
}

func main() {
	flag.Parse()
	configureLog()

	// Load configuration
	conf, err := configuration.FromFile(configFilePath)
	if err != nil {
		log.Fatal(err)
	}

	eventBus := event_bus.New()

	// Wait for died children to avoid zombies
	signalChannel := make(chan os.Signal, 2)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGCHLD)
	go func() {
		for {
			sig := <-signalChannel
			if sig == syscall.SIGCHLD {
				r := syscall.Rusage{}
				syscall.Wait4(-1, nil, 0, &r)
			}
		}
	}()

	// Create StatsD client
	conf.StatsD.CreateClient()

	// Create Zookeeper connection
	zkConn := listenToZookeeper(conf, eventBus)

	// Register handlers
	handlers := event_bus.Handlers{Conf: &conf, Zookeeper: zkConn}
	eventBus.Register(handlers.MarathonEventHandler)
	eventBus.Register(handlers.ServiceEventHandler)
	eventBus.Publish(event_bus.MarathonEvent{EventType: "bamboo_startup", Timestamp: time.Now().Format(time.RFC3339)})

	// Handle gracefully exit
	registerOSSignals()

	// Start server
	initServer(&conf, zkConn, eventBus)
}

func initServer(conf *configuration.Configuration, conn *zk.Conn, eventBus *event_bus.EventBus) {
	stateAPI := api.StateAPI{Config: conf, Zookeeper: conn}
	serviceAPI := api.ServiceAPI{Config: conf, Zookeeper: conn}
	eventSubAPI := api.EventSubscriptionAPI{Conf: conf, EventBus: eventBus}

	conf.StatsD.Increment(1.0, "restart", 1)
	// Status live information
	router := martini.Classic()
	router.Get("/status", api.HandleStatus)

	// API
	router.Group("/api", func(api martini.Router) {
		// State API
		api.Get("/state", stateAPI.Get)
		// Service API
		api.Get("/services", serviceAPI.All)
		api.Post("/services", serviceAPI.Create)
		api.Put("/services/**", serviceAPI.Put)
		api.Delete("/services/**", serviceAPI.Delete)
		api.Post("/marathon/event_callback", eventSubAPI.Callback)
	})

	// Static pages
	router.Use(martini.Static(path.Join(executableFolder(), "webapp")))

	if conf.Marathon.UseEventStream {
		// Listen events stream from Marathon
		listenToEventStream(conf, eventSubAPI)
	} else {
		registerMarathonEvent(conf)
	}
	router.RunOnAddr(serverBindPort)
}

// Get current executable folder path
func executableFolder() string {
	folderPath, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}
	return folderPath
}

func registerMarathonEvent(conf *configuration.Configuration) {

	client := &http.Client{}
	// it's safe to register with multiple marathon nodes
	for _, marathon := range conf.Marathon.Endpoints() {
		url := marathon + "/v2/eventSubscriptions?callbackUrl=" + conf.Bamboo.Endpoint + "/api/marathon/event_callback"
		req, _ := http.NewRequest("POST", url, nil)
		req.Header.Add("Content-Type", "application/json")
		resp, err := client.Do(req)
		if err != nil {
			errorMsg := "An error occurred while accessing Marathon callback system: %s\n"
			log.Printf(errorMsg, err)
			return
		}
		bodyBytes, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Fatal(err)
			return
		}
		body := string(bodyBytes)
		if strings.HasPrefix(body, "{\"message") {
			warningMsg := "Access to the callback system of Marathon seems to be failed, response: %s\n"
			log.Printf(warningMsg, body)
		}
	}
}

func createAndListen(conf configuration.Zookeeper) (chan zk.Event, *zk.Conn) {
	conn, _, err := zk.Connect(conf.ConnectionString(), time.Second*10)

	if err != nil {
		log.Panic(err)
	}

	ch, _ := qzk.ListenToConn(conn, conf.Path, true, conf.Delay())
	return ch, conn
}

func listenToZookeeper(conf configuration.Configuration, eventBus *event_bus.EventBus) *zk.Conn {
	serviceCh, serviceConn := createAndListen(conf.Bamboo.Zookeeper)

	go func() {
		for {
			select {
			case _ = <-serviceCh:
				eventBus.Publish(event_bus.ServiceEvent{EventType: "change"})
			}
		}
	}()
	return serviceConn
}

func listenToEventStream(conf *configuration.Configuration, sub api.EventSubscriptionAPI) {
	client := &http.Client{}
	client.Timeout = 0 * time.Second

	for _, marathon := range conf.Marathon.Endpoints() {
		ticker := time.NewTicker(1 * time.Second)
		go func() {
			for _ = range ticker.C {
				req, err := http.NewRequest("GET", marathon+"/v2/events", nil)
				req.Header.Set("Accept", "text/event-stream")
				if err != nil {
					errorMsg := "An error occurred while creating request to Marathon events system: %s\n"
					log.Printf(errorMsg, err)
					continue
				}

				resp, err := client.Do(req)
				if err != nil {
					errorMsg := "An error occurred while making a request to Marathon events system: %s\n"
					log.Printf(errorMsg, err)
					continue
				}

				defer resp.Body.Close()

				reader := bufio.NewReader(resp.Body)
				for {
					line, err := reader.ReadString('\n')
					if err != nil {
						if err != io.EOF {
							errorMsg := "An error occurred while reading Marathon event: %s\n"
							log.Printf(errorMsg, err)
						}
						break
					}

					if len(strings.TrimSpace(line)) == 0 {
						continue
					}

					if !strings.HasPrefix(line, "data: ") {
						errorMsg := "Wrong event format: %s\n"
						log.Printf(errorMsg, line)
						continue
					}

					line = line[6:]
					sub.Notify([]byte(line))
				}

				log.Println("Event stream connection was closed. Re-opening...")
			}
		}()
	}
}

func configureLog() {
	if len(logPath) > 0 {
		log.SetOutput(io.MultiWriter(&lumberjack.Logger{
			Filename: logPath,
			// megabytes
			MaxSize:    100,
			MaxBackups: 3,
			//days
			MaxAge: 28,
		}, os.Stdout))
	}
}

func registerOSSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			log.Println("Server Stopped")
			os.Exit(0)
		}
	}()
}
<commit_msg>re #149 rename 'listenToEventStream' to 'listenToMarathonEventStream'<commit_after>package main

import (
	"bufio"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"

	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/go-martini/martini"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/kardianos/osext"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/natefinch/lumberjack"
	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/samuel/go-zookeeper/zk"
	"github.com/QubitProducts/bamboo/api"
	"github.com/QubitProducts/bamboo/configuration"
	"github.com/QubitProducts/bamboo/qzk"
	"github.com/QubitProducts/bamboo/services/event_bus"
)

/*
	Commandline arguments
*/
var configFilePath string
var logPath string
var serverBindPort string

func init() {
	flag.StringVar(&configFilePath, "config", "config/development.json", "Full path of the configuration JSON file")
	flag.StringVar(&logPath, "log", "", "Log path to a file. Default logs to stdout")
	flag.StringVar(&serverBindPort, "bind", ":8000", "Bind HTTP server to a specific port")
}

func main() {
	flag.Parse()
	configureLog()

	// Load configuration
	conf, err := configuration.FromFile(configFilePath)
	if err != nil {
		log.Fatal(err)
	}

	eventBus := event_bus.New()

	// Wait for died children to avoid zombies
	signalChannel := make(chan os.Signal, 2)
	signal.Notify(signalChannel, os.Interrupt, syscall.SIGCHLD)
	go func() {
		for {
			sig := <-signalChannel
			if sig == syscall.SIGCHLD {
				r := syscall.Rusage{}
				syscall.Wait4(-1, nil, 0, &r)
			}
		}
	}()

	// Create StatsD client
	conf.StatsD.CreateClient()

	// Create Zookeeper connection
	zkConn := listenToZookeeper(conf, eventBus)

	// Register handlers
	handlers := event_bus.Handlers{Conf: &conf, Zookeeper: zkConn}
	eventBus.Register(handlers.MarathonEventHandler)
	eventBus.Register(handlers.ServiceEventHandler)
	eventBus.Publish(event_bus.MarathonEvent{EventType: "bamboo_startup", Timestamp: time.Now().Format(time.RFC3339)})

	// Handle gracefully exit
	registerOSSignals()

	// Start server
	initServer(&conf, zkConn, eventBus)
}

func initServer(conf *configuration.Configuration, conn *zk.Conn, eventBus *event_bus.EventBus) {
	stateAPI := api.StateAPI{Config: conf, Zookeeper: conn}
	serviceAPI := api.ServiceAPI{Config: conf, Zookeeper: conn}
	eventSubAPI := api.EventSubscriptionAPI{Conf: conf, EventBus: eventBus}

	conf.StatsD.Increment(1.0, "restart", 1)
	// Status live information
	router := martini.Classic()
	router.Get("/status", api.HandleStatus)

	// API
	router.Group("/api", func(api martini.Router) {
		// State API
		api.Get("/state", stateAPI.Get)
		// Service API
		api.Get("/services", serviceAPI.All)
		api.Post("/services", serviceAPI.Create)
		api.Put("/services/**", serviceAPI.Put)
		api.Delete("/services/**", serviceAPI.Delete)
		api.Post("/marathon/event_callback", eventSubAPI.Callback)
	})

	// Static pages
	router.Use(martini.Static(path.Join(executableFolder(), "webapp")))

	if conf.Marathon.UseEventStream {
		// Listen events stream from Marathon
		listenToMarathonEventStream(conf, eventSubAPI)
	} else {
		registerMarathonEvent(conf)
	}
	router.RunOnAddr(serverBindPort)
}

// Get current executable folder path
func executableFolder() string {
	folderPath, err := osext.ExecutableFolder()
	if err != nil {
		log.Fatal(err)
	}
	return folderPath
}

func registerMarathonEvent(conf *configuration.Configuration) {

	client := &http.Client{}
	// it's safe to register with multiple marathon nodes
	for _, marathon := range conf.Marathon.Endpoints() {
		url := marathon + "/v2/eventSubscriptions?callbackUrl=" + conf.Bamboo.Endpoint + "/api/marathon/event_callback"
		req, _ := http.NewRequest("POST", url, nil)
		req.Header.Add("Content-Type", "application/json")
		resp, err := client.Do(req)
		if err != nil {
			errorMsg := "An error occurred while accessing Marathon callback system: %s\n"
			log.Printf(errorMsg, err)
			return
		}
		bodyBytes, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			log.Fatal(err)
			return
		}
		body := string(bodyBytes)
		if strings.HasPrefix(body, "{\"message") {
			warningMsg := "Access to the callback system of Marathon seems to be failed, response: %s\n"
			log.Printf(warningMsg, body)
		}
	}
}

func createAndListen(conf configuration.Zookeeper) (chan zk.Event, *zk.Conn) {
	conn, _, err := zk.Connect(conf.ConnectionString(), time.Second*10)

	if err != nil {
		log.Panic(err)
	}

	ch, _ := qzk.ListenToConn(conn, conf.Path, true, conf.Delay())
	return ch, conn
}

func listenToZookeeper(conf configuration.Configuration, eventBus *event_bus.EventBus) *zk.Conn {
	serviceCh, serviceConn := createAndListen(conf.Bamboo.Zookeeper)

	go func() {
		for {
			select {
			case _ = <-serviceCh:
				eventBus.Publish(event_bus.ServiceEvent{EventType: "change"})
			}
		}
	}()
	return serviceConn
}

func listenToMarathonEventStream(conf *configuration.Configuration, sub api.EventSubscriptionAPI) {
	client := &http.Client{}
	client.Timeout = 0 * time.Second

	for _, marathon := range conf.Marathon.Endpoints() {
		ticker := time.NewTicker(1 * time.Second)
		go func() {
			for _ = range ticker.C {
				req, err := http.NewRequest("GET", marathon+"/v2/events", nil)
				req.Header.Set("Accept", "text/event-stream")
				if err != nil {
					errorMsg := "An error occurred while creating request to Marathon events system: %s\n"
					log.Printf(errorMsg, err)
					continue
				}

				resp, err := client.Do(req)
				if err != nil {
					errorMsg := "An error occurred while making a request to Marathon events system: %s\n"
					log.Printf(errorMsg, err)
					continue
				}

				defer resp.Body.Close()

				reader := bufio.NewReader(resp.Body)
				for {
					line, err := reader.ReadString('\n')
					if err != nil {
						if err != io.EOF {
							errorMsg := "An error occurred while reading Marathon event: %s\n"
							log.Printf(errorMsg, err)
						}
						break
					}

					if len(strings.TrimSpace(line)) == 0 {
						continue
					}

					if !strings.HasPrefix(line, "data: ") {
						errorMsg := "Wrong event format: %s\n"
						log.Printf(errorMsg, line)
						continue
					}

					line = line[6:]
					sub.Notify([]byte(line))
				}

				log.Println("Event stream connection was closed. Re-opening...")
			}
		}()
	}
}

func configureLog() {
	if len(logPath) > 0 {
		log.SetOutput(io.MultiWriter(&lumberjack.Logger{
			Filename: logPath,
			// megabytes
			MaxSize:    100,
			MaxBackups: 3,
			//days
			MaxAge: 28,
		}, os.Stdout))
	}
}

func registerOSSignals() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for _ = range c {
			log.Println("Server Stopped")
			os.Exit(0)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"math/rand"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/go-telegram-bot-api/telegram-bot-api"
	bt "github.com/ikool-cn/gobeanstalk-connection-pool"
)

var (
	telegramBot *TelegramBot
)

// TelegramBot ...
type TelegramBot struct {
	Name          string
	SelfChatID    int64
	ChannelChatID int64
	ComicPath     string
	DeleteDelay   time.Duration
	Client        *tgbotapi.BotAPI
	Queue         *bt.Pool
	Tube          string
}

// NewTelegramBot ...
func NewTelegramBot(cfg *TelegramConfig, btdAddr string) (t *TelegramBot) {
	bot, err := tgbotapi.NewBotAPI(cfg.Token)
	if err != nil {
		logger.Panicf("tg bot init failed: %+v", err)
	}
	delay, err := time.ParseDuration(cfg.DeleteDelay)
	if err != nil {
		logger.Panicf("delete delay error: %+v", err)
	}

	t = &TelegramBot{
		Name:          bot.Self.UserName,
		SelfChatID:    cfg.SelfChatID,
		ChannelChatID: cfg.ChannelChatID,
		ComicPath:     cfg.ComicPath,
		DeleteDelay:   delay,
		Client:        bot,
		Tube:          "tg",
	}
	t.Queue = &bt.Pool{
		Dial: func() (*bt.Conn, error) {
			return bt.Dial(btdAddr)
		},
		MaxIdle:     10,
		MaxActive:   100,
		IdleTimeout: 60 * time.Second,
		MaxLifetime: 180 * time.Second,
		Wait:        true,
	}
	return
}

func (t *TelegramBot) putQueue(msg []byte) {
	conn, err := t.Queue.Get()
	if err != nil {
		logger.Errorf("%+v: %s", err, string(msg))
		return
	}
	conn.Use(t.Tube)
	_, err = conn.Put(msg, 1, t.DeleteDelay, time.Minute)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
}

func (t *TelegramBot) sendFile(chat int64, file string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, file)
	return t.Client.Send(tgbotapi.NewDocumentUpload(chat, file))
}
func (t *TelegramBot) sendPic(chat int64, file string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, file)
	if strings.HasSuffix(file, ".mp4") {
		return t.Client.Send(tgbotapi.NewVideoUpload(chat, file))
	}
	return t.Client.Send(tgbotapi.NewPhotoUpload(chat, file))
}

func (t *TelegramBot) send(chat int64, msg string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, msg)
	return t.Client.Send(tgbotapi.NewMessage(chat, msg))
}

func (t *TelegramBot) delMessage() {
	for {
		conn, err := t.Queue.Get()
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		conn.Watch(t.Tube)
		job, err := conn.Reserve()
		if err != nil {
			logger.Warningf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		msg := &tgbotapi.Message{}
		err = json.Unmarshal(job.Body, msg)
		if err != nil {
			logger.Errorf("%+v", err)
			err = conn.Bury(job.ID, 0)
			if err != nil {
				logger.Errorf("%+v", err)
			}
			time.Sleep(3 * time.Second)
			continue
		}
		delMsg := tgbotapi.DeleteMessageConfig{
			ChatID:    msg.Chat.ID,
			MessageID: msg.MessageID,
		}
		logger.Infof(":[%s]{%s}", getMsgTitle(msg), strconv.Quote(msg.Text))

		_, err = t.Client.DeleteMessage(delMsg)
		if err != nil {
			logger.Errorf("%+v", err)
			err = conn.Bury(job.ID, 0)
			if err != nil {
				logger.Errorf("%+v", err)
			}
			time.Sleep(3 * time.Second)
			continue
		}
		err = conn.Delete(job.ID)
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
		}
		t.Queue.Release(conn, false)
	}
}

func (t *TelegramBot) tgBot() {
	u := tgbotapi.NewUpdate(0)
	u.Timeout = 60
	for {
		updates, err := t.Client.GetUpdatesChan(u)
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		var message *tgbotapi.Message
		for update := range updates {
			if update.Message != nil {
				message = update.Message
			} else if update.EditedMessage != nil {
				message = update.EditedMessage
			} else {
				// unkown msg type
				continue
			}
			if message.Chat.IsGroup() {
				logger.Infof(
					"recv:(%s)[%s]{%s}",
					message.Chat.Title,
					message.From.String(),
					strconv.Quote(message.Text))
			} else {
				logger.Infof(
					"recv:[%s]{%s}",
					message.From.String(),
					strconv.Quote(message.Text),
				)
			}

			if message.IsCommand() {
				switch message.Command() {
				case "start":
					go onStart(t, message)
				case "comic":
					go onComic(t, message)
				case "pic":
					go onPic(t, message)
				default:
					logger.Infof("ignore unkown cmd: %+v", message.Command())
					continue
				}
			} else {
				if message.Text == "" {
					continue
				}
				checkRepeat(t, message)
			}
		}
		logger.Warning("tg bot restarted.")
		time.Sleep(3 * time.Second)
	}
}

func checkRepeat(t *TelegramBot, message *tgbotapi.Message) {
	key := "tg_" + getMsgTitle(message) + "_last"
	flattendMsg := strings.TrimSpace(message.Text)
	defer redisClient.LTrim(key, 0, 10)
	defer redisClient.LPush(key, flattendMsg)

	lastMsgs, err := redisClient.LRange(key, 0, 6).Result()
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	i := 0
	for _, s := range lastMsgs {
		if s == flattendMsg {
			i++
		}
	}
	if i > 1 {
		redisClient.Del(key)
		logger.Infof("repeat: %s", strconv.Quote(message.Text))
		msg := tgbotapi.NewMessage(message.Chat.ID, message.Text)
		t.Client.Send(msg)
	}
}

func onStart(t *TelegramBot, message *tgbotapi.Message) {
	msg := tgbotapi.NewMessage(message.Chat.ID, "呀呀呀")
	msg.ReplyToMessageID = message.MessageID
	t.Client.Send(msg)
}

func onComic(t *TelegramBot, message *tgbotapi.Message) {
	files, err := filepath.Glob(t.ComicPath)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	rand.Seed(time.Now().Unix())
	file := files[rand.Intn(len(files))]
	number := strings.Split(strings.Split(file, "@")[1], ".")[0]
	msg := tgbotapi.NewMessage(message.Chat.ID, "https://nhentai.net/g/"+number)

	logger.Infof("send:[%s]{%s}", getMsgTitle(message), strconv.Quote(file))
	msgSent, err := t.Client.Send(msg)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	data, err := json.Marshal(msgSent)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	t.putQueue(data)
}

func onPic(t *TelegramBot, message *tgbotapi.Message) {
	files, err := filepath.Glob(twitterBot.ImgPath + "/*")
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	if files == nil {
		logger.Error("find no pics")
	}
	rand.Seed(time.Now().Unix())
	file := files[rand.Intn(len(files))]

	logger.Infof("send:[%s]{%s}", getMsgTitle(message), strconv.Quote(file))
	msgSent, err := t.sendFile(message.Chat.ID, file)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	data, err := json.Marshal(msgSent)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	// t.putQueue(data)
}

func getMsgTitle(m *tgbotapi.Message) string {
	if m.Chat.IsGroup() {
		return m.Chat.Title
	}
	return m.From.String()
}
<commit_msg>fix build<commit_after>package main

import (
	"encoding/json"
	"math/rand"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/go-telegram-bot-api/telegram-bot-api"
	bt "github.com/ikool-cn/gobeanstalk-connection-pool"
)

var (
	telegramBot *TelegramBot
)

// TelegramBot ...
type TelegramBot struct {
	Name          string
	SelfChatID    int64
	ChannelChatID int64
	ComicPath     string
	DeleteDelay   time.Duration
	Client        *tgbotapi.BotAPI
	Queue         *bt.Pool
	Tube          string
}

// NewTelegramBot ...
func NewTelegramBot(cfg *TelegramConfig, btdAddr string) (t *TelegramBot) {
	bot, err := tgbotapi.NewBotAPI(cfg.Token)
	if err != nil {
		logger.Panicf("tg bot init failed: %+v", err)
	}
	delay, err := time.ParseDuration(cfg.DeleteDelay)
	if err != nil {
		logger.Panicf("delete delay error: %+v", err)
	}

	t = &TelegramBot{
		Name:          bot.Self.UserName,
		SelfChatID:    cfg.SelfChatID,
		ChannelChatID: cfg.ChannelChatID,
		ComicPath:     cfg.ComicPath,
		DeleteDelay:   delay,
		Client:        bot,
		Tube:          "tg",
	}
	t.Queue = &bt.Pool{
		Dial: func() (*bt.Conn, error) {
			return bt.Dial(btdAddr)
		},
		MaxIdle:     10,
		MaxActive:   100,
		IdleTimeout: 60 * time.Second,
		MaxLifetime: 180 * time.Second,
		Wait:        true,
	}
	return
}

func (t *TelegramBot) putQueue(msg []byte) {
	conn, err := t.Queue.Get()
	if err != nil {
		logger.Errorf("%+v: %s", err, string(msg))
		return
	}
	conn.Use(t.Tube)
	_, err = conn.Put(msg, 1, t.DeleteDelay, time.Minute)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
}

func (t *TelegramBot) sendFile(chat int64, file string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, file)
	return t.Client.Send(tgbotapi.NewDocumentUpload(chat, file))
}
func (t *TelegramBot) sendPic(chat int64, file string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, file)
	if strings.HasSuffix(file, ".mp4") {
		return t.Client.Send(tgbotapi.NewVideoUpload(chat, file))
	}
	return t.Client.Send(tgbotapi.NewPhotoUpload(chat, file))
}

func (t *TelegramBot) send(chat int64, msg string) (tgbotapi.Message, error) {
	logger.Debugf("[%d]%s", chat, msg)
	return t.Client.Send(tgbotapi.NewMessage(chat, msg))
}

func (t *TelegramBot) delMessage() {
	for {
		conn, err := t.Queue.Get()
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		conn.Watch(t.Tube)
		job, err := conn.Reserve()
		if err != nil {
			logger.Warningf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		msg := &tgbotapi.Message{}
		err = json.Unmarshal(job.Body, msg)
		if err != nil {
			logger.Errorf("%+v", err)
			err = conn.Bury(job.ID, 0)
			if err != nil {
				logger.Errorf("%+v", err)
			}
			time.Sleep(3 * time.Second)
			continue
		}
		delMsg := tgbotapi.DeleteMessageConfig{
			ChatID:    msg.Chat.ID,
			MessageID: msg.MessageID,
		}
		logger.Infof(":[%s]{%s}", getMsgTitle(msg), strconv.Quote(msg.Text))

		_, err = t.Client.DeleteMessage(delMsg)
		if err != nil {
			logger.Errorf("%+v", err)
			err = conn.Bury(job.ID, 0)
			if err != nil {
				logger.Errorf("%+v", err)
			}
			time.Sleep(3 * time.Second)
			continue
		}
		err = conn.Delete(job.ID)
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
		}
		t.Queue.Release(conn, false)
	}
}

func (t *TelegramBot) tgBot() {
	u := tgbotapi.NewUpdate(0)
	u.Timeout = 60
	for {
		updates, err := t.Client.GetUpdatesChan(u)
		if err != nil {
			logger.Errorf("%+v", err)
			time.Sleep(3 * time.Second)
			continue
		}
		var message *tgbotapi.Message
		for update := range updates {
			if update.Message != nil {
				message = update.Message
			} else if update.EditedMessage != nil {
				message = update.EditedMessage
			} else {
				// unkown msg type
				continue
			}
			if message.Chat.IsGroup() {
				logger.Infof(
					"recv:(%s)[%s]{%s}",
					message.Chat.Title,
					message.From.String(),
					strconv.Quote(message.Text))
			} else {
				logger.Infof(
					"recv:[%s]{%s}",
					message.From.String(),
					strconv.Quote(message.Text),
				)
			}

			if message.IsCommand() {
				switch message.Command() {
				case "start":
					go onStart(t, message)
				case "comic":
					go onComic(t, message)
				case "pic":
					go onPic(t, message)
				default:
					logger.Infof("ignore unkown cmd: %+v", message.Command())
					continue
				}
			} else {
				if message.Text == "" {
					continue
				}
				checkRepeat(t, message)
			}
		}
		logger.Warning("tg bot restarted.")
		time.Sleep(3 * time.Second)
	}
}

func checkRepeat(t *TelegramBot, message *tgbotapi.Message) {
	key := "tg_" + getMsgTitle(message) + "_last"
	flattendMsg := strings.TrimSpace(message.Text)
	defer redisClient.LTrim(key, 0, 10)
	defer redisClient.LPush(key, flattendMsg)

	lastMsgs, err := redisClient.LRange(key, 0, 6).Result()
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	i := 0
	for _, s := range lastMsgs {
		if s == flattendMsg {
			i++
		}
	}
	if i > 1 {
		redisClient.Del(key)
		logger.Infof("repeat: %s", strconv.Quote(message.Text))
		msg := tgbotapi.NewMessage(message.Chat.ID, message.Text)
		t.Client.Send(msg)
	}
}

func onStart(t *TelegramBot, message *tgbotapi.Message) {
	msg := tgbotapi.NewMessage(message.Chat.ID, "呀呀呀")
	msg.ReplyToMessageID = message.MessageID
	t.Client.Send(msg)
}

func onComic(t *TelegramBot, message *tgbotapi.Message) {
	files, err := filepath.Glob(t.ComicPath)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	rand.Seed(time.Now().Unix())
	file := files[rand.Intn(len(files))]
	number := strings.Split(strings.Split(file, "@")[1], ".")[0]
	msg := tgbotapi.NewMessage(message.Chat.ID, "https://nhentai.net/g/"+number)

	logger.Infof("send:[%s]{%s}", getMsgTitle(message), strconv.Quote(file))
	msgSent, err := t.Client.Send(msg)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	data, err := json.Marshal(msgSent)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	t.putQueue(data)
}

func onPic(t *TelegramBot, message *tgbotapi.Message) {
	files, err := filepath.Glob(twitterBot.ImgPath + "/*")
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	if files == nil {
		logger.Error("find no pics")
	}
	rand.Seed(time.Now().Unix())
	file := files[rand.Intn(len(files))]

	logger.Infof("send:[%s]{%s}", getMsgTitle(message), strconv.Quote(file))
	_, err = t.sendFile(message.Chat.ID, file)
	if err != nil {
		logger.Errorf("%+v", err)
		return
	}
	// data, err := json.Marshal(msgSent)
	// if err != nil {
	// logger.Errorf("%+v", err)
	// return
	// }
	// t.putQueue(data)
}

func getMsgTitle(m *tgbotapi.Message) string {
	if m.Chat.IsGroup() {
		return m.Chat.Title
	}
	return m.From.String()
}
<|endoftext|>


<commit_before>package main

import "html/template"

var tmpl = template.Must(template.New("tmpl").Parse(`

	{{if .LawId}}{{.LawId}}{{else}}??-ФЗ{{end}}
	№{{.OrderId}}
	{{if .ExhibitionNumber}}
		Лот {{.ExhibitionNumber}}
	{{end}}

		{{if .OrderName}}{{.OrderName}}{{else}}unknown{{end}}

{{if .OKDP}}

ОКДП: {{.OKDP}}

{{end}}
{{if .OKPD}}

ОКПД: {{.OKPD}}

{{end}}

	Сроки подачи заявки:
	с
	{{if .StartDilingDate}}{{.StartDilingDate}}
	{{else}}00.00.0000{{end}}
	по
	<s>
		{{if .FinishDilingDate}}{{.FinishDilingDate}}
		{{else}}00.00.0000{{end}}
	</s>

	Начальная (максимальная) цена:
	{{.StartOrderPrice}}
	{{if .CurrencyId}}{{.CurrencyId}}{{else}}unknown currency{{end}}

Тип закупки: {{.OrderType}}

Этап закупки: {{.OrderStage}}

Дата публикации извещения: {{.PubDate}}

Организация: {{.OrganisationName}}

{{if .Features}}

{{.Features}}

{{end}}
{{if .Errors}}

		Ошибки при анализе закупки <s>(проверьте извещение)</s>:

		{{range .Errors}}

{{.}}

		{{end}}

{{end}}
`))
<commit_msg>sm fix<commit_after>package main

import "html/template"

var tmpl = template.Must(template.New("tmpl").Parse(`<!DOCTYPE html>

		<title>{{.Title}}</title>

			{{if .LawId}}{{.LawId}}{{else}}??-ФЗ{{end}}
			№{{.OrderId}}
			{{if .ExhibitionNumber}}
				Лот {{.ExhibitionNumber}}
			{{end}}

				{{if .OrderName}}{{.OrderName}}{{else}}unknown{{end}}

		{{if .OKDP}}

ОКДП: {{.OKDP}}

		{{end}}
		{{if .OKPD}}

ОКПД: {{.OKPD}}

		{{end}}

			Сроки подачи заявки:
			с
			{{if .StartDilingDate}}{{.StartDilingDate}}
			{{else}}00.00.0000{{end}}
			по
			<s>
				{{if .FinishDilingDate}}{{.FinishDilingDate}}
				{{else}}00.00.0000{{end}}
			</s>

			Начальная (максимальная) цена:
			{{.StartOrderPrice}}
			{{if .CurrencyId}}{{.CurrencyId}}
			{{else}}unknown currency{{end}}

Тип закупки: {{.OrderType}}

Этап закупки: {{.OrderStage}}

Дата публикации извещения: {{.PubDate}}

Организация: {{.OrganisationName}}

		{{if .Features}}

{{.Features}}

		{{end}}
		{{if .Errors}}

				<s>Проверьте извещение</s>, были обнаружены ошибки:

				{{range .Errors}}

{{.}}

				{{end}}

		{{end}}

`))
<|endoftext|>


<commit_before>package restic

import (
	"fmt"
	"io"
	"math/rand"
	"restic/backend"
	"restic/pack"
	"restic/repository"
	"testing"
	"time"

	"github.com/restic/chunker"
)

// fakeFile returns a reader which yields deterministic pseudo-random data.
func fakeFile(t testing.TB, seed, size int64) io.Reader {
	return io.LimitReader(repository.NewRandReader(rand.New(rand.NewSource(seed))), size)
}

// saveFile reads from rd and saves the blobs in the repository. The list of
// IDs is returned.
func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) {
	ch := chunker.New(rd, repo.Config.ChunkerPolynomial)

	for {
		chunk, err := ch.Next(getBuf())
		if err == io.EOF {
			break
		}

		if err != nil {
			t.Fatalf("unabel to save chunk in repo: %v", err)
		}

		id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
		if err != nil {
			t.Fatalf("error saving chunk: %v", err)
		}
		blobs = append(blobs, id)
	}

	return blobs
}

const (
	maxFileSize = 1500000
	maxSeed     = 20
	maxNodes    = 32
)

// saveTree saves a tree of fake files in the repo and returns the ID.
func saveTree(t testing.TB, repo *repository.Repository, seed int64, depth int) backend.ID {
	t.Logf("create fake tree with seed %d, depth %d", seed, depth)

	rnd := rand.NewSource(seed)
	numNodes := int(rnd.Int63() % maxNodes)
	t.Logf("create %v nodes", numNodes)

	var tree Tree
	for i := 0; i < numNodes; i++ {

		// randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4).
		if depth > 1 && rnd.Int63()%4 == 0 {
			treeSeed := rnd.Int63() % maxSeed
			id := saveTree(t, repo, treeSeed, depth-1)

			node := &Node{
				Name:    fmt.Sprintf("dir-%v", treeSeed),
				Type:    "dir",
				Mode:    0755,
				Subtree: &id,
			}

			tree.Nodes = append(tree.Nodes, node)
			continue
		}

		fileSeed := rnd.Int63() % maxSeed
		fileSize := rnd.Int63() % maxFileSize

		node := &Node{
			Name: fmt.Sprintf("file-%v", fileSeed),
			Type: "file",
			Mode: 0644,
			Size: uint64(fileSize),
		}

		node.Content = saveFile(t, repo, fakeFile(t, fileSeed, fileSize))
		tree.Nodes = append(tree.Nodes, node)
	}

	id, err := repo.SaveJSON(pack.Tree, tree)
	if err != nil {
		t.Fatal(err)
	}

	return id
}

// TestCreateSnapshot creates a snapshot filled with fake data. The
// fake data is generated deterministically from the timestamp `at`, which is
// also used as the snapshot's timestamp. The tree's depth can be specified
// with the parameter depth.
func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time, depth int) backend.ID {
	seed := at.Unix()
	t.Logf("create fake snapshot at %s with seed %d", at, seed)

	fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
	snapshot, err := NewSnapshot([]string{fakedir})
	if err != nil {
		t.Fatal(err)
	}
	snapshot.Time = at

	treeID := saveTree(t, repo, seed, depth)
	snapshot.Tree = &treeID

	id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("saved snapshot %v", id.Str())

	err = repo.Flush()
	if err != nil {
		t.Fatal(err)
	}

	err = repo.SaveIndex()
	if err != nil {
		t.Fatal(err)
	}

	return id
}
<commit_msg>Make test files in test repo less random<commit_after>package restic

import (
	"fmt"
	"io"
	"math/rand"
	"restic/backend"
	"restic/pack"
	"restic/repository"
	"testing"
	"time"

	"github.com/restic/chunker"
)

// fakeFile returns a reader which yields deterministic pseudo-random data.
func fakeFile(t testing.TB, seed, size int64) io.Reader {
	return io.LimitReader(repository.NewRandReader(rand.New(rand.NewSource(seed))), size)
}

// saveFile reads from rd and saves the blobs in the repository. The list of
// IDs is returned.
func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) {
	blobs = backend.IDs{}
	ch := chunker.New(rd, repo.Config.ChunkerPolynomial)

	for {
		chunk, err := ch.Next(getBuf())
		if err == io.EOF {
			break
		}

		if err != nil {
			t.Fatalf("unable to save chunk in repo: %v", err)
		}

		id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
		if err != nil {
			t.Fatalf("error saving chunk: %v", err)
		}
		blobs = append(blobs, id)
	}

	return blobs
}

const (
	maxFileSize = 1500000
	maxSeed     = 32
	maxNodes    = 32
)

// saveTree saves a tree of fake files in the repo and returns the ID.
func saveTree(t testing.TB, repo *repository.Repository, seed int64, depth int) backend.ID {
	t.Logf("create fake tree with seed %d, depth %d", seed, depth)

	rnd := rand.NewSource(seed)
	numNodes := int(rnd.Int63() % maxNodes)
	t.Logf("create %v nodes", numNodes)

	var tree Tree
	for i := 0; i < numNodes; i++ {

		// randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4).
		if depth > 1 && rnd.Int63()%4 == 0 {
			treeSeed := rnd.Int63() % maxSeed
			id := saveTree(t, repo, treeSeed, depth-1)

			node := &Node{
				Name:    fmt.Sprintf("dir-%v", treeSeed),
				Type:    "dir",
				Mode:    0755,
				Subtree: &id,
			}

			tree.Nodes = append(tree.Nodes, node)
			continue
		}

		fileSeed := rnd.Int63() % maxSeed
		fileSize := (maxFileSize / maxSeed) * fileSeed

		node := &Node{
			Name: fmt.Sprintf("file-%v", fileSeed),
			Type: "file",
			Mode: 0644,
			Size: uint64(fileSize),
		}

		node.Content = saveFile(t, repo, fakeFile(t, fileSeed, fileSize))
		tree.Nodes = append(tree.Nodes, node)
	}

	id, err := repo.SaveJSON(pack.Tree, tree)
	if err != nil {
		t.Fatal(err)
	}

	return id
}

// TestCreateSnapshot creates a snapshot filled with fake data. The
// fake data is generated deterministically from the timestamp `at`, which is
// also used as the snapshot's timestamp. The tree's depth can be specified
// with the parameter depth.
func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time, depth int) backend.ID {
	seed := at.Unix()
	t.Logf("create fake snapshot at %s with seed %d", at, seed)

	fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
	snapshot, err := NewSnapshot([]string{fakedir})
	if err != nil {
		t.Fatal(err)
	}
	snapshot.Time = at

	treeID := saveTree(t, repo, seed, depth)
	snapshot.Tree = &treeID

	id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("saved snapshot %v", id.Str())

	err = repo.Flush()
	if err != nil {
		t.Fatal(err)
	}

	err = repo.SaveIndex()
	if err != nil {
		t.Fatal(err)
	}

	return id
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Cgo; see gmp.go for an overview.

// TODO(rsc):
//	Emit correct line number annotations.
//	Make 6g understand the annotations.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strings"
)

// A Package collects information about the package we're going to write.
type Package struct {
	PackageName string // name of package
	PackagePath string
	PtrSize     int64
	IntSize     int64
	GccOptions  []string
	CgoFlags    map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
	Written     map[string]bool
	Name        map[string]*Name // accumulated Name from Files
	ExpFunc     []*ExpFunc       // accumulated ExpFunc from Files
	Decl        []ast.Decl
	GoFiles     []string // list of Go files
	GccFiles    []string // list of gcc output files
	Preamble    string   // collected preamble for _cgo_export.h
}

// A File collects information about a single Go input file.
type File struct {
	AST      *ast.File           // parsed AST
	Comments []*ast.CommentGroup // comments from file
	Package  string              // Package name
	Preamble string              // C preamble (doc comment on import "C")
	Ref      []*Ref              // all references to C.xxx in AST
	ExpFunc  []*ExpFunc          // exported functions for this file
	Name     map[string]*Name    // map from Go name to Name
}

func nameKeys(m map[string]*Name) []string {
	var ks []string
	for k := range m {
		ks = append(ks, k)
	}
	sort.Strings(ks)
	return ks
}

// A Ref refers to an expression of the form C.xxx in the AST.
type Ref struct {
	Name    *Name
	Expr    *ast.Expr
	Context string // "type", "expr", "call", or "call2"
}

func (r *Ref) Pos() token.Pos {
	return (*r.Expr).Pos()
}

// A Name collects information about C.xxx.
type Name struct {
	Go       string // name used in Go referring to package C
	Mangle   string // name used in generated Go
	C        string // name used in C
	Define   string // #define expansion
	Kind     string // "const", "type", "var", "fpvar", "func", "not-type"
	Type     *Type  // the type of xxx
	FuncType *FuncType
	AddError bool
	Const    string // constant definition
}

// IsVar returns true if Kind is either "var" or "fpvar"
func (n *Name) IsVar() bool {
	return n.Kind == "var" || n.Kind == "fpvar"
}

// A ExpFunc is an exported function, callable from C.
// Such functions are identified in the Go input file
// by doc comments containing the line //export ExpName
type ExpFunc struct {
	Func    *ast.FuncDecl
	ExpName string // name to use from C
}

// A TypeRepr contains the string representation of a type.
type TypeRepr struct {
	Repr       string
	FormatArgs []interface{}
}

// A Type collects information about a type in both the C and Go worlds.
type Type struct {
	Size       int64
	Align      int64
	C          *TypeRepr
	Go         ast.Expr
	EnumValues map[string]int64
	Typedef    string
}

// A FuncType collects information about a function type in both the C and Go worlds.
type FuncType struct {
	Params []*Type
	Result *Type
	Go     *ast.FuncType
}

func usage() {
	fmt.Fprint(os.Stderr, "usage: cgo -- [compiler options] file.go ...\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var ptrSizeMap = map[string]int64{
	"386":     4,
	"amd64":   8,
	"arm":     4,
	"ppc64":   8,
	"ppc64le": 8,
	"s390":    4,
	"s390x":   8,
}

var intSizeMap = map[string]int64{
	"386":     4,
	"amd64":   8,
	"arm":     4,
	"ppc64":   8,
	"ppc64le": 8,
	"s390":    4,
	"s390x":   4,
}

var cPrefix string

var fset = token.NewFileSet()

var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
var dynout = flag.String("dynout", "", "write -dynimport output to this file")
var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
var dynlinker = flag.Bool("dynlinker", false, "record dynamic linker information in -dynimport mode")

// These flags are for bootstrapping a new Go implementation,
// to generate Go types that match the data layout and
// constant values used in the host's C libraries and system calls.
var godefs = flag.Bool("godefs", false, "for bootstrap: write Go definitions for C file to standard output")
var objDir = flag.String("objdir", "", "object directory")

var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
var goarch, goos string

func main() {
	flag.Usage = usage
	flag.Parse()

	if *dynobj != "" {
		// cgo -dynimport is essentially a separate helper command
		// built into the cgo binary.  It scans a gcc-produced executable
		// and dumps information about the imported symbols and the
		// imported libraries.  The 'go build' rules for cgo prepare an
		// appropriate executable and then use its import information
		// instead of needing to make the linkers duplicate all the
		// specialized knowledge gcc has about where to look for imported
		// symbols and which ones to use.
		dynimport(*dynobj)
		return
	}

	if *godefs {
		// Generating definitions pulled from header files,
		// to be checked into Go repositories.
		// Line numbers are just noise.
		conf.Mode &^= printer.SourcePos
	}

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	// Find first arg that looks like a go file and assume everything before
	// that are options to pass to gcc.
	var i int
	for i = len(args); i > 0; i-- {
		if !strings.HasSuffix(args[i-1], ".go") {
			break
		}
	}
	if i == len(args) {
		usage()
	}

	goFiles := args[i:]

	p := newPackage(args[:i])

	// Record CGO_LDFLAGS from the environment for external linking.
	if ldflags := os.Getenv("CGO_LDFLAGS"); ldflags != "" {
		args, err := splitQuoted(ldflags)
		if err != nil {
			fatalf("bad CGO_LDFLAGS: %q (%s)", ldflags, err)
		}
		p.addToFlag("LDFLAGS", args)
	}

	// Need a unique prefix for the global C symbols that
	// we use to coordinate between gcc and ourselves.
	// We already put _cgo_ at the beginning, so the main
	// concern is other cgo wrappers for the same functions.
	// Use the beginning of the md5 of the input to disambiguate.
	h := md5.New()
	for _, input := range goFiles {
		f, err := os.Open(input)
		if err != nil {
			fatalf("%s", err)
		}
		io.Copy(h, f)
		f.Close()
	}
	cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6])

	fs := make([]*File, len(goFiles))
	for i, input := range goFiles {
		f := new(File)
		f.ReadGo(input)
		f.DiscardCgoDirectives()
		fs[i] = f
	}

	if *objDir == "" {
		// make sure that _obj directory exists, so that we can write
		// all the output files there.
		os.Mkdir("_obj", 0777)
		*objDir = "_obj"
	}
	*objDir += string(filepath.Separator)

	for i, input := range goFiles {
		f := fs[i]
		p.Translate(f)
		for _, cref := range f.Ref {
			switch cref.Context {
			case "call", "call2":
				if cref.Name.Kind != "type" {
					break
				}
				*cref.Expr = cref.Name.Type.Go
			}
		}
		if nerrors > 0 {
			os.Exit(2)
		}
		pkg := f.Package
		if dir := os.Getenv("CGOPKGPATH"); dir != "" {
			pkg = filepath.Join(dir, pkg)
		}
		p.PackagePath = pkg
		p.Record(f)
		if *godefs {
			os.Stdout.WriteString(p.godefs(f, input))
		} else {
			p.writeOutput(f, input)
		}
	}

	if !*godefs {
		p.writeDefs()
	}
	if nerrors > 0 {
		os.Exit(2)
	}
}

// newPackage returns a new Package that will invoke
// gcc with the additional arguments specified in args.
func newPackage(args []string) *Package {
	goarch = runtime.GOARCH
	if s := os.Getenv("GOARCH"); s != "" {
		goarch = s
	}
	goos = runtime.GOOS
	if s := os.Getenv("GOOS"); s != "" {
		goos = s
	}
	ptrSize := ptrSizeMap[goarch]
	if ptrSize == 0 {
		fatalf("unknown ptrSize for $GOARCH %q", goarch)
	}
	intSize := intSizeMap[goarch]
	if intSize == 0 {
		fatalf("unknown intSize for $GOARCH %q", goarch)
	}

	// Reset locale variables so gcc emits English errors [sic].
	os.Setenv("LANG", "en_US.UTF-8")
	os.Setenv("LC_ALL", "C")

	p := &Package{
		PtrSize:  ptrSize,
		IntSize:  intSize,
		CgoFlags: make(map[string][]string),
		Written:  make(map[string]bool),
	}
	p.addToFlag("CFLAGS", args)
	return p
}

// Record what needs to be recorded about f.
func (p *Package) Record(f *File) {
	if p.PackageName == "" {
		p.PackageName = f.Package
	} else if p.PackageName != f.Package {
		error_(token.NoPos, "inconsistent package names: %s, %s", p.PackageName, f.Package)
	}

	if p.Name == nil {
		p.Name = f.Name
	} else {
		for k, v := range f.Name {
			if p.Name[k] == nil {
				p.Name[k] = v
			} else if !reflect.DeepEqual(p.Name[k], v) {
				error_(token.NoPos, "inconsistent definitions for C.%s", fixGo(k))
			}
		}
	}

	if f.ExpFunc != nil {
		p.ExpFunc = append(p.ExpFunc, f.ExpFunc...)
		p.Preamble += "\n" + f.Preamble
	}
	p.Decl = append(p.Decl, f.AST.Decls...)
}
<commit_msg>cmd/cgo: add support for GOARCH=arm64<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Cgo; see gmp.go for an overview.

// TODO(rsc):
//	Emit correct line number annotations.
//	Make 6g understand the annotations.

package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"go/ast"
	"go/printer"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strings"
)

// A Package collects information about the package we're going to write.
type Package struct {
	PackageName string // name of package
	PackagePath string
	PtrSize     int64
	IntSize     int64
	GccOptions  []string
	CgoFlags    map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
	Written     map[string]bool
	Name        map[string]*Name // accumulated Name from Files
	ExpFunc     []*ExpFunc       // accumulated ExpFunc from Files
	Decl        []ast.Decl
	GoFiles     []string // list of Go files
	GccFiles    []string // list of gcc output files
	Preamble    string   // collected preamble for _cgo_export.h
}

// A File collects information about a single Go input file.
type File struct {
	AST      *ast.File           // parsed AST
	Comments []*ast.CommentGroup // comments from file
	Package  string              // Package name
	Preamble string              // C preamble (doc comment on import "C")
	Ref      []*Ref              // all references to C.xxx in AST
	ExpFunc  []*ExpFunc          // exported functions for this file
	Name     map[string]*Name    // map from Go name to Name
}

func nameKeys(m map[string]*Name) []string {
	var ks []string
	for k := range m {
		ks = append(ks, k)
	}
	sort.Strings(ks)
	return ks
}

// A Ref refers to an expression of the form C.xxx in the AST.
type Ref struct {
	Name    *Name
	Expr    *ast.Expr
	Context string // "type", "expr", "call", or "call2"
}

func (r *Ref) Pos() token.Pos {
	return (*r.Expr).Pos()
}

// A Name collects information about C.xxx.
type Name struct {
	Go       string // name used in Go referring to package C
	Mangle   string // name used in generated Go
	C        string // name used in C
	Define   string // #define expansion
	Kind     string // "const", "type", "var", "fpvar", "func", "not-type"
	Type     *Type  // the type of xxx
	FuncType *FuncType
	AddError bool
	Const    string // constant definition
}

// IsVar returns true if Kind is either "var" or "fpvar"
func (n *Name) IsVar() bool {
	return n.Kind == "var" || n.Kind == "fpvar"
}

// A ExpFunc is an exported function, callable from C.
// Such functions are identified in the Go input file
// by doc comments containing the line //export ExpName
type ExpFunc struct {
	Func    *ast.FuncDecl
	ExpName string // name to use from C
}

// A TypeRepr contains the string representation of a type.
type TypeRepr struct {
	Repr       string
	FormatArgs []interface{}
}

// A Type collects information about a type in both the C and Go worlds.
type Type struct {
	Size       int64
	Align      int64
	C          *TypeRepr
	Go         ast.Expr
	EnumValues map[string]int64
	Typedef    string
}

// A FuncType collects information about a function type in both the C and Go worlds.
type FuncType struct {
	Params []*Type
	Result *Type
	Go     *ast.FuncType
}

func usage() {
	fmt.Fprint(os.Stderr, "usage: cgo -- [compiler options] file.go ...\n")
	flag.PrintDefaults()
	os.Exit(2)
}

var ptrSizeMap = map[string]int64{
	"386":     4,
	"amd64":   8,
	"arm":     4,
	"arm64":   8,
	"ppc64":   8,
	"ppc64le": 8,
	"s390":    4,
	"s390x":   8,
}

var intSizeMap = map[string]int64{
	"386":     4,
	"amd64":   8,
	"arm":     4,
	"arm64":   8,
	"ppc64":   8,
	"ppc64le": 8,
	"s390":    4,
	"s390x":   4,
}

var cPrefix string

var fset = token.NewFileSet()

var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
var dynout = flag.String("dynout", "", "write -dynimport output to this file")
var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
var dynlinker = flag.Bool("dynlinker", false, "record dynamic linker information in -dynimport mode")

// These flags are for bootstrapping a new Go implementation,
// to generate Go types that match the data layout and
// constant values used in the host's C libraries and system calls.
var godefs = flag.Bool("godefs", false, "for bootstrap: write Go definitions for C file to standard output")
var objDir = flag.String("objdir", "", "object directory")

var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
var goarch, goos string

func main() {
	flag.Usage = usage
	flag.Parse()

	if *dynobj != "" {
		// cgo -dynimport is essentially a separate helper command
		// built into the cgo binary.  It scans a gcc-produced executable
		// and dumps information about the imported symbols and the
		// imported libraries.  The 'go build' rules for cgo prepare an
		// appropriate executable and then use its import information
		// instead of needing to make the linkers duplicate all the
		// specialized knowledge gcc has about where to look for imported
		// symbols and which ones to use.
		dynimport(*dynobj)
		return
	}

	if *godefs {
		// Generating definitions pulled from header files,
		// to be checked into Go repositories.
		// Line numbers are just noise.
		conf.Mode &^= printer.SourcePos
	}

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	// Find first arg that looks like a go file and assume everything before
	// that are options to pass to gcc.
	var i int
	for i = len(args); i > 0; i-- {
		if !strings.HasSuffix(args[i-1], ".go") {
			break
		}
	}
	if i == len(args) {
		usage()
	}

	goFiles := args[i:]

	p := newPackage(args[:i])

	// Record CGO_LDFLAGS from the environment for external linking.
	if ldflags := os.Getenv("CGO_LDFLAGS"); ldflags != "" {
		args, err := splitQuoted(ldflags)
		if err != nil {
			fatalf("bad CGO_LDFLAGS: %q (%s)", ldflags, err)
		}
		p.addToFlag("LDFLAGS", args)
	}

	// Need a unique prefix for the global C symbols that
	// we use to coordinate between gcc and ourselves.
	// We already put _cgo_ at the beginning, so the main
	// concern is other cgo wrappers for the same functions.
	// Use the beginning of the md5 of the input to disambiguate.
	h := md5.New()
	for _, input := range goFiles {
		f, err := os.Open(input)
		if err != nil {
			fatalf("%s", err)
		}
		io.Copy(h, f)
		f.Close()
	}
	cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6])

	fs := make([]*File, len(goFiles))
	for i, input := range goFiles {
		f := new(File)
		f.ReadGo(input)
		f.DiscardCgoDirectives()
		fs[i] = f
	}

	if *objDir == "" {
		// make sure that _obj directory exists, so that we can write
		// all the output files there.
		os.Mkdir("_obj", 0777)
		*objDir = "_obj"
	}
	*objDir += string(filepath.Separator)

	for i, input := range goFiles {
		f := fs[i]
		p.Translate(f)
		for _, cref := range f.Ref {
			switch cref.Context {
			case "call", "call2":
				if cref.Name.Kind != "type" {
					break
				}
				*cref.Expr = cref.Name.Type.Go
			}
		}
		if nerrors > 0 {
			os.Exit(2)
		}
		pkg := f.Package
		if dir := os.Getenv("CGOPKGPATH"); dir != "" {
			pkg = filepath.Join(dir, pkg)
		}
		p.PackagePath = pkg
		p.Record(f)
		if *godefs {
			os.Stdout.WriteString(p.godefs(f, input))
		} else {
			p.writeOutput(f, input)
		}
	}

	if !*godefs {
		p.writeDefs()
	}
	if nerrors > 0 {
		os.Exit(2)
	}
}

// newPackage returns a new Package that will invoke
// gcc with the additional arguments specified in args.
func newPackage(args []string) *Package {
	goarch = runtime.GOARCH
	if s := os.Getenv("GOARCH"); s != "" {
		goarch = s
	}
	goos = runtime.GOOS
	if s := os.Getenv("GOOS"); s != "" {
		goos = s
	}
	ptrSize := ptrSizeMap[goarch]
	if ptrSize == 0 {
		fatalf("unknown ptrSize for $GOARCH %q", goarch)
	}
	intSize := intSizeMap[goarch]
	if intSize == 0 {
		fatalf("unknown intSize for $GOARCH %q", goarch)
	}

	// Reset locale variables so gcc emits English errors [sic].
	os.Setenv("LANG", "en_US.UTF-8")
	os.Setenv("LC_ALL", "C")

	p := &Package{
		PtrSize:  ptrSize,
		IntSize:  intSize,
		CgoFlags: make(map[string][]string),
		Written:  make(map[string]bool),
	}
	p.addToFlag("CFLAGS", args)
	return p
}

// Record what needs to be recorded about f.
func (p *Package) Record(f *File) {
	if p.PackageName == "" {
		p.PackageName = f.Package
	} else if p.PackageName != f.Package {
		error_(token.NoPos, "inconsistent package names: %s, %s", p.PackageName, f.Package)
	}

	if p.Name == nil {
		p.Name = f.Name
	} else {
		for k, v := range f.Name {
			if p.Name[k] == nil {
				p.Name[k] = v
			} else if !reflect.DeepEqual(p.Name[k], v) {
				error_(token.NoPos, "inconsistent definitions for C.%s", fixGo(k))
			}
		}
	}

	if f.ExpFunc != nil {
		p.ExpFunc = append(p.ExpFunc, f.ExpFunc...)
		p.Preamble += "\n" + f.Preamble
	}
	p.Decl = append(p.Decl, f.AST.Decls...)
}
<|endoftext|>


<commit_before>package lager

import (
	"bytes"
	"encoding/json"
	"io"
	"sync"
)

type TestSink struct {
	contents []byte
	lock     *sync.Mutex
}

func NewTestSink() *TestSink {
	return &TestSink{
		lock: &sync.Mutex{},
	}
}

func (l *TestSink) Log(level LogLevel, p []byte) {
	l.lock.Lock()
	defer l.lock.Unlock()

	l.contents = append(l.contents, p...)
}

func (l *TestSink) Buffer() *bytes.Buffer {
	l.lock.Lock()
	defer l.lock.Unlock()

	contents := make([]byte, len(l.contents))
	copy(contents, l.contents)
	return bytes.NewBuffer(contents)
}

func (l *TestSink) Logs() []LogFormat {
	logs := []LogFormat{}
	decoder := json.NewDecoder(l.Buffer())
	for {
		var log LogFormat
		if err := decoder.Decode(&log); err == io.EOF {
			return logs
		} else if err != nil {
			panic(err)
		}
		logs = append(logs, log)
	}
	return logs
}
<commit_msg>add convenient test logger<commit_after>package lager

import (
	"bytes"
	"encoding/json"
	"io"
	"sync"
)

type TestLogger struct {
	Logger
	*TestSink
}

func NewTestLogger(component string) *TestLogger {
	logger := NewLogger(component)
	testSink := NewTestSink()
	logger.RegisterSink(testSink)

	return &TestLogger{logger, testSink}
}

type TestSink struct {
	contents []byte
	lock     *sync.Mutex
}

func NewTestSink() *TestSink {
	return &TestSink{
		lock: &sync.Mutex{},
	}
}

func (l *TestSink) Log(level LogLevel, p []byte) {
	l.lock.Lock()
	defer l.lock.Unlock()

	l.contents = append(l.contents, p...)
}

func (l *TestSink) Buffer() *bytes.Buffer {
	l.lock.Lock()
	defer l.lock.Unlock()

	contents := make([]byte, len(l.contents))
	copy(contents, l.contents)
	return bytes.NewBuffer(contents)
}

func (l *TestSink) Logs() []LogFormat {
	logs := []LogFormat{}
	decoder := json.NewDecoder(l.Buffer())
	for {
		var log LogFormat
		if err := decoder.Decode(&log); err == io.EOF {
			return logs
		} else if err != nil {
			panic(err)
		}
		logs = append(logs, log)
	}
	return logs
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Vet is a simple checker for static errors in Go source code.
// See doc.go for more information.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var verbose = flag.Bool("v", false, "verbose")
var exitCode = 0

// Flags to control which checks to perform. "all" is set to true here, and disabled later if
// a flag is set explicitly.
var report = map[string]*bool{
	"all":        flag.Bool("all", true, "check everything; disabled if any explicit check is requested"),
	"atomic":     flag.Bool("atomic", false, "check for common mistaken usages of the sync/atomic package"),
	"buildtags":  flag.Bool("buildtags", false, "check that +build tags are valid"),
	"composites": flag.Bool("composites", false, "check that composite literals used type-tagged elements"),
	"methods":    flag.Bool("methods", false, "check that canonically named methods are canonically defined"),
	"printf":     flag.Bool("printf", false, "check printf-like invocations"),
	"structtags": flag.Bool("structtags", false, "check that struct field tags have canonical format"),
	"rangeloops": flag.Bool("rangeloops", false, "check that range loop variables are used correctly"),
}

// vet tells whether to report errors for the named check, a flag name.
func vet(name string) bool {
	return *report["all"] || *report[name]
}

// setExit sets the value for os.Exit when it is called, later.  It
// remembers the highest value.
func setExit(err int) {
	if err > exitCode {
		exitCode = err
	}
}

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\tvet [flags] directory...\n")
	fmt.Fprintf(os.Stderr, "\tvet [flags] files... # Must be a single package\n")
	flag.PrintDefaults()
	os.Exit(2)
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	pkg  *Package
	fset *token.FileSet
	name string
	file *ast.File
	b    bytes.Buffer // for use by methods
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	// If a check is named explicitly, turn off the 'all' flag.
	for name, ptr := range report {
		if name != "all" && *ptr {
			*report["all"] = false
			break
		}
	}

	if *printfuncs != "" {
		for _, name := range strings.Split(*printfuncs, ",") {
			if len(name) == 0 {
				flag.Usage()
			}
			skip := 0
			if colon := strings.LastIndex(name, ":"); colon > 0 {
				var err error
				skip, err = strconv.Atoi(name[colon+1:])
				if err != nil {
					errorf(`illegal format for "Func:N" argument %q; %s`, name, err)
				}
				name = name[:colon]
			}
			name = strings.ToLower(name)
			if name[len(name)-1] == 'f' {
				printfList[name] = skip
			} else {
				printList[name] = skip
			}
		}
	}

	if flag.NArg() == 0 {
		Usage()
	}
	dirs := false
	files := false
	for _, name := range flag.Args() {
		// Is it a directory?
		fi, err := os.Stat(name)
		if err != nil {
			warnf("error walking tree: %s", err)
			continue
		}
		if fi.IsDir() {
			dirs = true
		} else {
			files = true
		}
	}
	if dirs && files {
		Usage()
	}
	if dirs {
		for _, name := range flag.Args() {
			walkDir(name)
		}
		return
	}
	doPackage(flag.Args())
	os.Exit(exitCode)
}

// doPackageDir analyzes the single package found in the directory, if there is one.
func doPackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		// If it's just that there are no go source files, that's fine.
		if _, nogo := err.(*build.NoGoError); nogo {
			return
		}
		// Non-fatal: we are doing a recursive walk and there may be other directories.
		warnf("cannot process directory %s: %s", directory, err)
		return
	}
	names := append(pkg.GoFiles, pkg.CgoFiles...)
	// Prefix file names with directory names.
	if directory != "." {
		for i, name := range names {
			names[i] = filepath.Join(directory, name)
		}
	}
	doPackage(names)
}

type Package struct {
	types  map[ast.Expr]types.Type
	values map[ast.Expr]interface{}
}

// doPackage analyzes the single package constructed from the named files.
func doPackage(names []string) {
	var files []*File
	var astFiles []*ast.File
	fs := token.NewFileSet()
	for _, name := range names {
		f, err := os.Open(name)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		defer f.Close()
		data, err := ioutil.ReadAll(f)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		checkBuildTag(name, data)
		parsedFile, err := parser.ParseFile(fs, name, bytes.NewReader(data), 0)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		files = append(files, &File{fset: fs, name: name, file: parsedFile})
		astFiles = append(astFiles, parsedFile)
	}
	pkg := new(Package)
	pkg.types = make(map[ast.Expr]types.Type)
	pkg.values = make(map[ast.Expr]interface{})
	exprFn := func(x ast.Expr, typ types.Type, val interface{}) {
		pkg.types[x] = typ
		if val != nil {
			pkg.values[x] = val
		}
	}
	context := types.Context{
		Expr: exprFn,
	}
	// Type check the package.
	_, err := context.Check(fs, astFiles)
	if err != nil {
		warnf("%s", err)
	}
	for _, file := range files {
		file.pkg = pkg
		file.walkFile(file.name, file.file)
	}
}

func visit(path string, f os.FileInfo, err error) error {
	if err != nil {
		errorf("walk error: %s", err)
	}
	// One package per directory. Ignore the files themselves.
	if !f.IsDir() {
		return nil
	}
	doPackageDir(path)
	return nil
}

// walkDir recursively walks the tree looking for .go files.
func walkDir(root string) {
	filepath.Walk(root, visit)
}

// errorf formats the error to standard error, adding program
// identification and a newline, and exits.
func errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...)
	os.Exit(2)
}

// warnf formats the error to standard error, adding program
// identification and a newline, but does not exit.
func warnf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...)
	setExit(1)
}

// Println is fmt.Println guarded by -v.
func Println(args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Println(args...)
}

// Printf is fmt.Printf guarded by -v.
func Printf(format string, args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Printf(format+"\n", args...)
}

// Bad reports an error and sets the exit code..
func (f *File) Bad(pos token.Pos, args ...interface{}) {
	f.Warn(pos, args...)
	setExit(1)
}

// Badf reports a formatted error and sets the exit code.
func (f *File) Badf(pos token.Pos, format string, args ...interface{}) {
	f.Warnf(pos, format, args...)
	setExit(1)
}

func (f *File) loc(pos token.Pos) string {
	// Do not print columns. Because the pos often points to the start of an
	// expression instead of the inner part with the actual error, the
	// precision can mislead.
	posn := f.fset.Position(pos)
	return fmt.Sprintf("%s:%d: ", posn.Filename, posn.Line)
}

// Warn reports an error but does not set the exit code.
func (f *File) Warn(pos token.Pos, args ...interface{}) {
	fmt.Fprint(os.Stderr, f.loc(pos)+fmt.Sprintln(args...))
}

// Warnf reports a formatted error but does not set the exit code.
func (f *File) Warnf(pos token.Pos, format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, f.loc(pos)+format+"\n", args...)
}

// walkFile walks the file's tree.
func (f *File) walkFile(name string, file *ast.File) {
	Println("Checking file", name)
	ast.Walk(f, file)
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.AssignStmt:
		f.walkAssignStmt(n)
	case *ast.CallExpr:
		f.walkCallExpr(n)
	case *ast.CompositeLit:
		f.walkCompositeLit(n)
	case *ast.Field:
		f.walkFieldTag(n)
	case *ast.FuncDecl:
		f.walkMethodDecl(n)
	case *ast.InterfaceType:
		f.walkInterfaceType(n)
	case *ast.RangeStmt:
		f.walkRangeStmt(n)
	}
	return f
}

// walkAssignStmt walks an assignment statement
func (f *File) walkAssignStmt(stmt *ast.AssignStmt) {
	f.checkAtomicAssignment(stmt)
}

// walkCall walks a call expression.
func (f *File) walkCall(call *ast.CallExpr, name string) {
	f.checkFmtPrintfCall(call, name)
}

// walkCallExpr walks a call expression.
func (f *File) walkCallExpr(call *ast.CallExpr) {
	switch x := call.Fun.(type) {
	case *ast.Ident:
		f.walkCall(call, x.Name)
	case *ast.SelectorExpr:
		f.walkCall(call, x.Sel.Name)
	}
}

// walkCompositeLit walks a composite literal.
func (f *File) walkCompositeLit(c *ast.CompositeLit) {
	f.checkUntaggedLiteral(c)
}

// walkFieldTag walks a struct field tag.
func (f *File) walkFieldTag(field *ast.Field) {
	if field.Tag == nil {
		return
	}
	f.checkCanonicalFieldTag(field)
}

// walkMethodDecl walks the method's signature.
func (f *File) walkMethod(id *ast.Ident, t *ast.FuncType) {
	f.checkCanonicalMethod(id, t)
}

// walkMethodDecl walks the method signature in the declaration.
func (f *File) walkMethodDecl(d *ast.FuncDecl) {
	if d.Recv == nil {
		// not a method
		return
	}
	f.walkMethod(d.Name, d.Type)
}

// walkInterfaceType walks the method signatures of an interface.
func (f *File) walkInterfaceType(t *ast.InterfaceType) {
	for _, field := range t.Methods.List {
		for _, id := range field.Names {
			f.walkMethod(id, field.Type.(*ast.FuncType))
		}
	}
}

// walkRangeStmt walks a range statement.
func (f *File) walkRangeStmt(n *ast.RangeStmt) {
	checkRangeLoop(f, n)
}

// goFmt returns a string representation of the expression
func (f *File) gofmt(x ast.Expr) string {
	f.b.Reset()
	printer.Fprint(&f.b, f.fset, x)
	return f.b.String()
}
<commit_msg>cmd/vet: silence error from type checker unless verbose is set. Also restores the checking of _test.go files, which disappeared as a result of the package-at-a-time change. Fixes issue 4895.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Vet is a simple checker for static errors in Go source code.
// See doc.go for more information.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"go/types"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var verbose = flag.Bool("v", false, "verbose")
var exitCode = 0

// Flags to control which checks to perform. "all" is set to true here, and disabled later if
// a flag is set explicitly.
var report = map[string]*bool{
	"all":        flag.Bool("all", true, "check everything; disabled if any explicit check is requested"),
	"atomic":     flag.Bool("atomic", false, "check for common mistaken usages of the sync/atomic package"),
	"buildtags":  flag.Bool("buildtags", false, "check that +build tags are valid"),
	"composites": flag.Bool("composites", false, "check that composite literals used type-tagged elements"),
	"methods":    flag.Bool("methods", false, "check that canonically named methods are canonically defined"),
	"printf":     flag.Bool("printf", false, "check printf-like invocations"),
	"structtags": flag.Bool("structtags", false, "check that struct field tags have canonical format"),
	"rangeloops": flag.Bool("rangeloops", false, "check that range loop variables are used correctly"),
}

// vet tells whether to report errors for the named check, a flag name.
func vet(name string) bool {
	return *report["all"] || *report[name]
}

// setExit sets the value for os.Exit when it is called, later.  It
// remembers the highest value.
func setExit(err int) {
	if err > exitCode {
		exitCode = err
	}
}

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\tvet [flags] directory...\n")
	fmt.Fprintf(os.Stderr, "\tvet [flags] files... # Must be a single package\n")
	flag.PrintDefaults()
	os.Exit(2)
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	pkg  *Package
	fset *token.FileSet
	name string
	file *ast.File
	b    bytes.Buffer // for use by methods
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	// If a check is named explicitly, turn off the 'all' flag.
	for name, ptr := range report {
		if name != "all" && *ptr {
			*report["all"] = false
			break
		}
	}

	if *printfuncs != "" {
		for _, name := range strings.Split(*printfuncs, ",") {
			if len(name) == 0 {
				flag.Usage()
			}
			skip := 0
			if colon := strings.LastIndex(name, ":"); colon > 0 {
				var err error
				skip, err = strconv.Atoi(name[colon+1:])
				if err != nil {
					errorf(`illegal format for "Func:N" argument %q; %s`, name, err)
				}
				name = name[:colon]
			}
			name = strings.ToLower(name)
			if name[len(name)-1] == 'f' {
				printfList[name] = skip
			} else {
				printList[name] = skip
			}
		}
	}

	if flag.NArg() == 0 {
		Usage()
	}
	dirs := false
	files := false
	for _, name := range flag.Args() {
		// Is it a directory?
		fi, err := os.Stat(name)
		if err != nil {
			warnf("error walking tree: %s", err)
			continue
		}
		if fi.IsDir() {
			dirs = true
		} else {
			files = true
		}
	}
	if dirs && files {
		Usage()
	}
	if dirs {
		for _, name := range flag.Args() {
			walkDir(name)
		}
		return
	}
	doPackage(flag.Args())
	os.Exit(exitCode)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) {
	if directory != "." {
		for i, name := range names {
			names[i] = filepath.Join(directory, name)
		}
	}
}

// doPackageDir analyzes the single package found in the directory, if there is one,
// plus a test package, if there is one.
func doPackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		// If it's just that there are no go source files, that's fine.
		if _, nogo := err.(*build.NoGoError); nogo {
			return
		}
		// Non-fatal: we are doing a recursive walk and there may be other directories.
		warnf("cannot process directory %s: %s", directory, err)
		return
	}
	var names []string
	names = append(names, pkg.CgoFiles...)
	names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package.
	prefixDirectory(directory, names)
	doPackage(names)
	// Is there also a "foo_test" package? If so, do that one as well.
	if len(pkg.XTestGoFiles) > 0 {
		names = pkg.XTestGoFiles
		prefixDirectory(directory, names)
		doPackage(names)
	}
}

type Package struct {
	types  map[ast.Expr]types.Type
	values map[ast.Expr]interface{}
}

// doPackage analyzes the single package constructed from the named files.
func doPackage(names []string) {
	var files []*File
	var astFiles []*ast.File
	fs := token.NewFileSet()
	for _, name := range names {
		f, err := os.Open(name)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		defer f.Close()
		data, err := ioutil.ReadAll(f)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		checkBuildTag(name, data)
		parsedFile, err := parser.ParseFile(fs, name, bytes.NewReader(data), 0)
		if err != nil {
			errorf("%s: %s", name, err)
		}
		files = append(files, &File{fset: fs, name: name, file: parsedFile})
		astFiles = append(astFiles, parsedFile)
	}
	pkg := new(Package)
	pkg.types = make(map[ast.Expr]types.Type)
	pkg.values = make(map[ast.Expr]interface{})
	exprFn := func(x ast.Expr, typ types.Type, val interface{}) {
		pkg.types[x] = typ
		if val != nil {
			pkg.values[x] = val
		}
	}
	context := types.Context{
		Expr: exprFn,
	}
	// Type check the package.
	_, err := context.Check(fs, astFiles)
	if err != nil && *verbose {
		warnf("%s", err)
	}
	for _, file := range files {
		file.pkg = pkg
		file.walkFile(file.name, file.file)
	}
}

func visit(path string, f os.FileInfo, err error) error {
	if err != nil {
		errorf("walk error: %s", err)
	}
	// One package per directory. Ignore the files themselves.
	if !f.IsDir() {
		return nil
	}
	doPackageDir(path)
	return nil
}

// walkDir recursively walks the tree looking for .go files.
func walkDir(root string) {
	filepath.Walk(root, visit)
}

// errorf formats the error to standard error, adding program
// identification and a newline, and exits.
func errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...)
	os.Exit(2)
}

// warnf formats the error to standard error, adding program
// identification and a newline, but does not exit.
func warnf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "vet: "+format+"\n", args...)
	setExit(1)
}

// Println is fmt.Println guarded by -v.
func Println(args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Println(args...)
}

// Printf is fmt.Printf guarded by -v.
func Printf(format string, args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Printf(format+"\n", args...)
}

// Bad reports an error and sets the exit code..
func (f *File) Bad(pos token.Pos, args ...interface{}) {
	f.Warn(pos, args...)
	setExit(1)
}

// Badf reports a formatted error and sets the exit code.
func (f *File) Badf(pos token.Pos, format string, args ...interface{}) {
	f.Warnf(pos, format, args...)
	setExit(1)
}

func (f *File) loc(pos token.Pos) string {
	// Do not print columns. Because the pos often points to the start of an
	// expression instead of the inner part with the actual error, the
	// precision can mislead.
	posn := f.fset.Position(pos)
	return fmt.Sprintf("%s:%d: ", posn.Filename, posn.Line)
}

// Warn reports an error but does not set the exit code.
func (f *File) Warn(pos token.Pos, args ...interface{}) {
	fmt.Fprint(os.Stderr, f.loc(pos)+fmt.Sprintln(args...))
}

// Warnf reports a formatted error but does not set the exit code.
func (f *File) Warnf(pos token.Pos, format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, f.loc(pos)+format+"\n", args...)
}

// walkFile walks the file's tree.
func (f *File) walkFile(name string, file *ast.File) {
	Println("Checking file", name)
	ast.Walk(f, file)
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	switch n := node.(type) {
	case *ast.AssignStmt:
		f.walkAssignStmt(n)
	case *ast.CallExpr:
		f.walkCallExpr(n)
	case *ast.CompositeLit:
		f.walkCompositeLit(n)
	case *ast.Field:
		f.walkFieldTag(n)
	case *ast.FuncDecl:
		f.walkMethodDecl(n)
	case *ast.InterfaceType:
		f.walkInterfaceType(n)
	case *ast.RangeStmt:
		f.walkRangeStmt(n)
	}
	return f
}

// walkAssignStmt walks an assignment statement
func (f *File) walkAssignStmt(stmt *ast.AssignStmt) {
	f.checkAtomicAssignment(stmt)
}

// walkCall walks a call expression.
func (f *File) walkCall(call *ast.CallExpr, name string) {
	f.checkFmtPrintfCall(call, name)
}

// walkCallExpr walks a call expression.
func (f *File) walkCallExpr(call *ast.CallExpr) {
	switch x := call.Fun.(type) {
	case *ast.Ident:
		f.walkCall(call, x.Name)
	case *ast.SelectorExpr:
		f.walkCall(call, x.Sel.Name)
	}
}

// walkCompositeLit walks a composite literal.
func (f *File) walkCompositeLit(c *ast.CompositeLit) {
	f.checkUntaggedLiteral(c)
}

// walkFieldTag walks a struct field tag.
func (f *File) walkFieldTag(field *ast.Field) {
	if field.Tag == nil {
		return
	}
	f.checkCanonicalFieldTag(field)
}

// walkMethodDecl walks the method's signature.
func (f *File) walkMethod(id *ast.Ident, t *ast.FuncType) {
	f.checkCanonicalMethod(id, t)
}

// walkMethodDecl walks the method signature in the declaration.
func (f *File) walkMethodDecl(d *ast.FuncDecl) {
	if d.Recv == nil {
		// not a method
		return
	}
	f.walkMethod(d.Name, d.Type)
}

// walkInterfaceType walks the method signatures of an interface.
func (f *File) walkInterfaceType(t *ast.InterfaceType) {
	for _, field := range t.Methods.List {
		for _, id := range field.Names {
			f.walkMethod(id, field.Type.(*ast.FuncType))
		}
	}
}

// walkRangeStmt walks a range statement.
func (f *File) walkRangeStmt(n *ast.RangeStmt) {
	checkRangeLoop(f, n)
}

// goFmt returns a string representation of the expression
func (f *File) gofmt(x ast.Expr) string {
	f.b.Reset()
	printer.Fprint(&f.b, f.fset, x)
	return f.b.String()
}
<|endoftext|>


<commit_before>package cassandra_store

import (
	"fmt"
	"strings"
	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"

	"github.com/gocql/gocql"
)

/*

Basically you need a table just like this:

CREATE TABLE seaweed_files (
   path varchar,
   fids list<varchar>,
   PRIMARY KEY (path)
);

Need to match flat_namespace.FlatNamespaceStore interface
	Put(fullFileName string, fid string) (err error)
	Get(fullFileName string) (fid string, err error)
	Delete(fullFileName string) (fid string, err error)

*/
type CassandraStore struct {
	cluster *gocql.ClusterConfig
	session *gocql.Session
}

func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) {
	c = &CassandraStore{}
	        s := strings.Split(hosts, ",")
        if len(s) == 1 {
		glog.V(2).Info("Only one  cassandra node to connect!A Cluster is Proposed!Now using:", string(hosts))
                c.cluster = gocql.NewCluster(hosts)
        } else if len(s) > 1 {
                c.cluster = gocql.NewCluster(s...)
        }
	c.cluster.Keyspace = keyspace
	c.cluster.Consistency = gocql.Quorum
	c.session, err = c.cluster.CreateSession()
	if err != nil {
		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
	}
	return
}

func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
	var input []string
	input = append(input, fid)
	if err := c.session.Query(
		`INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
		fullFileName, input).Exec(); err != nil {
		glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
		return err
	}
	return nil
}
func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
	var output []string
	if err := c.session.Query(
		`select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
		fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
		if err != gocql.ErrNotFound {
			glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
			return "", filer.ErrNotFound
		}
	}
	if len(output) == 0 {
		return "", fmt.Errorf("No file id found for %s", fullFileName)
	}
	return output[0], nil
}

// Currently the fid is not returned
func (c *CassandraStore) Delete(fullFileName string) (err error) {
	if err := c.session.Query(
		`DELETE FROM seaweed_files WHERE path = ?`,
		fullFileName).Exec(); err != nil {
		if err != gocql.ErrNotFound {
			glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
		}
		return err
	}
	return nil
}

func (c *CassandraStore) Close() {
	if c.session != nil {
		c.session.Close()
	}
}
<commit_msg>fix text.<commit_after>package cassandra_store

import (
	"fmt"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"

	"github.com/gocql/gocql"
)

/*

Basically you need a table just like this:

CREATE TABLE seaweed_files (
   path varchar,
   fids list<varchar>,
   PRIMARY KEY (path)
);

Need to match flat_namespace.FlatNamespaceStore interface
	Put(fullFileName string, fid string) (err error)
	Get(fullFileName string) (fid string, err error)
	Delete(fullFileName string) (fid string, err error)

*/
type CassandraStore struct {
	cluster *gocql.ClusterConfig
	session *gocql.Session
}

func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) {
	c = &CassandraStore{}
	s := strings.Split(hosts, ",")
	if len(s) == 1 {
		glog.V(2).Info("Only one cassandra node to connect! A cluster is Recommended! Now using:", string(hosts))
		c.cluster = gocql.NewCluster(hosts)
	} else if len(s) > 1 {
		c.cluster = gocql.NewCluster(s...)
	}
	c.cluster.Keyspace = keyspace
	c.cluster.Consistency = gocql.Quorum
	c.session, err = c.cluster.CreateSession()
	if err != nil {
		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
	}
	return
}

func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
	var input []string
	input = append(input, fid)
	if err := c.session.Query(
		`INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
		fullFileName, input).Exec(); err != nil {
		glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
		return err
	}
	return nil
}
func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
	var output []string
	if err := c.session.Query(
		`select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
		fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
		if err != gocql.ErrNotFound {
			glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
			return "", filer.ErrNotFound
		}
	}
	if len(output) == 0 {
		return "", fmt.Errorf("No file id found for %s", fullFileName)
	}
	return output[0], nil
}

// Currently the fid is not returned
func (c *CassandraStore) Delete(fullFileName string) (err error) {
	if err := c.session.Query(
		`DELETE FROM seaweed_files WHERE path = ?`,
		fullFileName).Exec(); err != nil {
		if err != gocql.ErrNotFound {
			glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
		}
		return err
	}
	return nil
}

func (c *CassandraStore) Close() {
	if c.session != nil {
		c.session.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package imagemetadataworker

import (
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/api/imagemetadata"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	environsmetadata "github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.imagemetadataworker")

// updatePublicImageMetadataPeriod is how frequently we check for
// public image metadata updates.
const updatePublicImageMetadataPeriod = time.Hour * 24

// ListPublishedMetadataFunc is the type of a function that is supplied to
// NewWorker for listing environment-specific published images metadata.
type ListPublishedMetadataFunc func(env environs.Environ) ([]*environsmetadata.ImageMetadata, error)

// DefaultListBlockDevices is the default function for listing block
// devices for the operating system of the local host.
var DefaultListPublishedMetadata ListPublishedMetadataFunc

func init() {
	DefaultListPublishedMetadata = list
}

// NewWorker returns a worker that lists published cloud
// images metadata, and records them in state.
func NewWorker(cl *imagemetadata.Client, l ListPublishedMetadataFunc, env environs.Environ) worker.Worker {
	f := func(stop <-chan struct{}) error {
		return doWork(cl, l, env)
	}
	return worker.NewPeriodicWorker(f, updatePublicImageMetadataPeriod, worker.NewTimer)
}

func doWork(cl *imagemetadata.Client, listf ListPublishedMetadataFunc, env environs.Environ) error {
	published, err := listf(env)
	if err != nil {
		return errors.Annotatef(err, "getting published images metadata")
	}
	err = save(cl, published)
	return errors.Annotatef(err, "saving published images metadata")
}

func list(env environs.Environ) ([]*environsmetadata.ImageMetadata, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	// We want all metadata, hence empty constraints.
	cons := environsmetadata.ImageConstraint{}
	metadata, _, err := environsmetadata.Fetch(sources, &cons, false)
	if err != nil {
		return nil, err
	}
	return metadata, nil
}

func save(client *imagemetadata.Client, published []*environsmetadata.ImageMetadata) error {
	// Store converted metadata.Note that whether the metadata actually needs
	// to be stored will be determined within this call.
	errs, err := client.Save(convertToParams(published))
	if err != nil {
		return errors.Annotatef(err, "saving published images metadata")
	}
	return processErrors(errs)
}

// convertToParams converts environment-specific images metadata to structured metadata format.
var convertToParams = func(published []*environsmetadata.ImageMetadata) []params.CloudImageMetadata {
	metadata := make([]params.CloudImageMetadata, len(published))
	for i, p := range published {
		metadata[i] = params.CloudImageMetadata{
			Source:          "public",
			ImageId:         p.Id,
			Stream:          p.Stream,
			Region:          p.RegionName,
			Arch:            p.Arch,
			VirtualType:     p.VirtType,
			RootStorageType: p.Storage,
		}
		// Translate version (eg.14.04) to a series (eg. "trusty")
		metadata[i].Series = versionSeries(p.Version)
	}

	return metadata
}

var seriesVersion = version.SeriesVersion

func versionSeries(v string) string {
	if v == "" {
		return v
	}
	for _, s := range version.SupportedSeries() {
		sv, err := seriesVersion(s)
		if err != nil {
			logger.Errorf("cannot determine version for series %v: %v", s, err)
		}
		if v == sv {
			return s
		}
	}
	return v
}

func processErrors(errs []params.ErrorResult) error {
	msgs := []string{}
	for _, e := range errs {
		if e.Error != nil && e.Error.Message != "" {
			msgs = append(msgs, e.Error.Message)
		}
	}
	if len(msgs) != 0 {
		return errors.Errorf("saving some image metadata:\n%v", strings.Join(msgs, "\n"))
	}
	return nil
}
<commit_msg>Filed a bug and added a TODO that references it.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package imagemetadataworker

import (
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/juju/loggo"

	"github.com/juju/juju/api/imagemetadata"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	environsmetadata "github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/version"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.worker.imagemetadataworker")

// updatePublicImageMetadataPeriod is how frequently we check for
// public image metadata updates.
const updatePublicImageMetadataPeriod = time.Hour * 24

// ListPublishedMetadataFunc is the type of a function that is supplied to
// NewWorker for listing environment-specific published images metadata.
type ListPublishedMetadataFunc func(env environs.Environ) ([]*environsmetadata.ImageMetadata, error)

// DefaultListBlockDevices is the default function for listing block
// devices for the operating system of the local host.
var DefaultListPublishedMetadata ListPublishedMetadataFunc

func init() {
	DefaultListPublishedMetadata = list
}

// NewWorker returns a worker that lists published cloud
// images metadata, and records them in state.
func NewWorker(cl *imagemetadata.Client, l ListPublishedMetadataFunc, env environs.Environ) worker.Worker {
	// TODO (anastasiamac 2015-09-02) Bug#1491353 - don't ignore stop channel.
	f := func(stop <-chan struct{}) error {
		return doWork(cl, l, env)
	}
	return worker.NewPeriodicWorker(f, updatePublicImageMetadataPeriod, worker.NewTimer)
}

func doWork(cl *imagemetadata.Client, listf ListPublishedMetadataFunc, env environs.Environ) error {
	published, err := listf(env)
	if err != nil {
		return errors.Annotatef(err, "getting published images metadata")
	}
	err = save(cl, published)
	return errors.Annotatef(err, "saving published images metadata")
}

func list(env environs.Environ) ([]*environsmetadata.ImageMetadata, error) {
	sources, err := environs.ImageMetadataSources(env)
	if err != nil {
		return nil, err
	}

	// We want all metadata, hence empty constraints.
	cons := environsmetadata.ImageConstraint{}
	metadata, _, err := environsmetadata.Fetch(sources, &cons, false)
	if err != nil {
		return nil, err
	}
	return metadata, nil
}

func save(client *imagemetadata.Client, published []*environsmetadata.ImageMetadata) error {
	// Store converted metadata.Note that whether the metadata actually needs
	// to be stored will be determined within this call.
	errs, err := client.Save(convertToParams(published))
	if err != nil {
		return errors.Annotatef(err, "saving published images metadata")
	}
	return processErrors(errs)
}

// convertToParams converts environment-specific images metadata to structured metadata format.
var convertToParams = func(published []*environsmetadata.ImageMetadata) []params.CloudImageMetadata {
	metadata := make([]params.CloudImageMetadata, len(published))
	for i, p := range published {
		metadata[i] = params.CloudImageMetadata{
			Source:          "public",
			ImageId:         p.Id,
			Stream:          p.Stream,
			Region:          p.RegionName,
			Arch:            p.Arch,
			VirtualType:     p.VirtType,
			RootStorageType: p.Storage,
		}
		// Translate version (eg.14.04) to a series (eg. "trusty")
		metadata[i].Series = versionSeries(p.Version)
	}

	return metadata
}

var seriesVersion = version.SeriesVersion

func versionSeries(v string) string {
	if v == "" {
		return v
	}
	for _, s := range version.SupportedSeries() {
		sv, err := seriesVersion(s)
		if err != nil {
			logger.Errorf("cannot determine version for series %v: %v", s, err)
		}
		if v == sv {
			return s
		}
	}
	return v
}

func processErrors(errs []params.ErrorResult) error {
	msgs := []string{}
	for _, e := range errs {
		if e.Error != nil && e.Error.Message != "" {
			msgs = append(msgs, e.Error.Message)
		}
	}
	if len(msgs) != 0 {
		return errors.Errorf("saving some image metadata:\n%v", strings.Join(msgs, "\n"))
	}
	return nil
}
<|endoftext|>