<commit_before>/*
 *        Copyright (C) 2014 Stefan 'glaxx' Luecke
 *
 *        This program is free software: you can redistribute it and/or modify
 *        it under the terms of the GNU Affero General Public License as published
 *        by the Free Software Foundation, either version 3 of the License, or
 *        (at your option) any later version.
 *
 *        This program is distributed in the hope that it will be useful,
 *        but WITHOUT ANY WARRANTY; without even the implied warranty of
 *        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *        GNU Affero General Public License for more details.
 *
 *        You should have received a copy of the GNU Affero General Public License
 *        along with this program.      If not, see <http://www.gnu.org/licenses/>.
 *
 *        Authors:	Stefan Luecke <glaxx@glaxx.net>
 */

 /*
 * TODO List:
 * - Test leap year behavior
 * - Test
 */

package cron

import (
//	"fmt"
	"time"
//	"sort"
	"log"
	"container/list"
)

type crontime struct {
	second []int
	minute []int
	hour   []int
	dow    []int //Day of Week
	dom    []int //Day of Month
	month  []int
	calculatedTime time.Time
	calculationInProgress bool
	eventList list.List
}

func (c *crontime) DurationUntilNextEvent() time.Duration {
	return c.nextEvent().Sub(time.Now())
}

func (c *crontime) GetNextEvent() time.Time {
	return c.eventList.Front().Value.(time.Time)
}

func (c *crontime) nextEvent() time.Time {
	if !c.calculationInProgress && c.eventList.Len() == 0{
		r := c.calculateEvent(time.Now())
		go c.fillList(r)
		return r
	} else if c.calculationInProgress && c.eventList.Len() == 0{
		// shit just got real aka TODO
		panic("Shit")

	} else if c.eventList.Len() > 0 {
		e := c.eventList.Front()
		r := e.Value.(time.Time)
		c.eventList.Remove(e)
		go c.fillList(c.eventList.Back().Value.(time.Time))
		return r
	}
	panic("shit 2")
}

func (c *crontime) fillList(baseTime time.Time) {
	if c.eventList.Len() == 0 {
		c.eventList.PushBack(c.calculateEvent(baseTime))
	}
	for ; c.eventList.Len() < 5; {
		c.eventList.PushBack(c.calculateEvent(c.eventList.Back().Value.(time.Time)))
	}
}

func (c *crontime) setCalculationInProgress(set bool) {
	c.calculationInProgress = set
}

// This functions calculates the next event
func (c *crontime) calculateEvent(baseTime time.Time) time.Time{
	c.calculationInProgress = true
	defer c.setCalculationInProgress(false)
	baseTime = setNanoecond(baseTime, 10000)
	c.calculatedTime = baseTime // Ignore all Events in the Past & initial 'result'
	//c.calculatedTime = setNanoecond(c.calculatedTime, 10000)
	c.nextValidMonth(baseTime)
	c.nextValidDay(baseTime)
	c.nextValidHour(baseTime)
	c.nextValidMinute(baseTime)
	c.nextValidSecond(baseTime)
	log.Println("Cronbee has found a time stamp: ", c.calculatedTime)
	return c.calculatedTime
}

// Calculates the next valid Month based upon the previous results.
func (c *crontime) nextValidMonth(baseTime time.Time) {
	for _, mon := range c.month {
		if mon >= int(c.calculatedTime.Month()) {
			//log.Print("Inside Month", mon, c.calculatedTime)
			c.calculatedTime = setMonth(c.calculatedTime, mon)
			//log.Println(" :: and out", c.calculatedTime)
			return
		}
	}
	// If no result was found try it again in the following year
	c.calculatedTime = c.calculatedTime.AddDate(1, 0, 0)
	c.calculatedTime = setMonth(c.calculatedTime, c.month[0])
	//log.Println("Cronbee: Month", c.calculatedTime, baseTime, c.month)
	c.nextValidMonth(baseTime)
}

// Calculates the next valid Day based upon the previous results.
func (c *crontime) nextValidDay(baseTime time.Time) {
	for _, dom := range c.dom {
		if dom >= c.calculatedTime.Day() {
			for _, dow := range c.dow {
				if monthHasDow(dow, dom, int(c.calculatedTime.Month()), c.calculatedTime.Year()){
					c.calculatedTime = setDay(c.calculatedTime, dom)
					//log.Println("Cronbee: Day-INS-1:", c.calculatedTime)
					return
				}
			}
		}
	}/* else {
		for _, dow := range c.dow {
			if monthHasDow(dow, dom, int(c.calculatedTime.Month()), c.calculatedTime.Year()){
				c.calculatedTime = setDay(c.calculatedTime, dom)
				log.Println("Cronbee: Day-INS-2:", c.calculatedTime)
				return
			}
		}
	}*/
	// If no result was found try it again in the following month.
	c.calculatedTime = c.calculatedTime.AddDate(0, 1, 0)
	c.calculatedTime = setDay(c.calculatedTime, c.dom[0])
	//log.Println("Cronbee: Day", c.calculatedTime, baseTime)
	c.nextValidMonth(baseTime)
	c.nextValidDay(baseTime)
}

// Calculates the next valid Hour based upon the previous results.
func (c *crontime) nextValidHour(baseTime time.Time) {
	for _, hour := range c.hour {
		if c.calculatedTime.Day() == baseTime.Day() {
			if !hasPassed(hour, c.calculatedTime.Hour()) {
				c.calculatedTime = setHour(c.calculatedTime, hour)
				return
			}
		} else {
			c.calculatedTime = setHour(c.calculatedTime, hour)
			return
		}
	}
	// If no result was found try it again in the following day.
	c.calculatedTime = c.calculatedTime.AddDate(0, 0, 1)     // <-|
	c.calculatedTime = setHour(c.calculatedTime, c.hour[0])  //   |
	//log.Println("Cronbee: Hour", c.calculatedTime, baseTime) // |
	c.nextValidMonth(baseTime) // May trigger a new month --------|
	c.nextValidDay(baseTime)
	c.nextValidHour(baseTime)
}

// Calculates the next valid Minute based upon the previous results.
func (c *crontime) nextValidMinute(baseTime time.Time) {
	for _, min := range c.minute {
		if c.calculatedTime.Hour() == baseTime.Hour() {
			if !hasPassed(min, c.calculatedTime.Minute()) {
				c.calculatedTime = setMinute(c.calculatedTime, min)
				return
			}
		} else {
			c.calculatedTime = setMinute(c.calculatedTime, min)
			return
		}
	}
	c.calculatedTime = c.calculatedTime.Add(1 * time.Hour)
	c.calculatedTime = setMinute(c.calculatedTime, c.minute[0])
	//log.Println("Cronbee: Minute", c.calculatedTime, baseTime)
	c.nextValidHour(baseTime)
	c.nextValidMinute(baseTime)
}

// Calculates the next valid Second based upon the previous results.
func (c *crontime) nextValidSecond(baseTime time.Time) {
	for _, sec := range c.second {
		if !c.minuteHasPassed(baseTime) {
			// check if sec is in the past. <= prevents triggering the same event twice
			if sec > c.calculatedTime.Second() {
				c.calculatedTime = setSecond(c.calculatedTime, sec)
				return
			}
		} else {
			c.calculatedTime = setSecond(c.calculatedTime, sec)
			return
		}
	}
	c.calculatedTime = c.calculatedTime.Add(1 * time.Minute)
	c.calculatedTime = setSecond(c.calculatedTime, 0)
	//log.Println("Cronbee: Second", c.calculatedTime, baseTime)
	c.nextValidMinute(baseTime)
	c.nextValidSecond(baseTime)
}

func (c *crontime) minuteHasPassed(baseTime time.Time) bool {
	if c.calculatedTime.Year() > baseTime.Year() {
		return true
	} else if c.calculatedTime.Month() > baseTime.Month() {
		return true
	} else if c.calculatedTime.Day() > baseTime.Day() {
		return true
	} else if c.calculatedTime.Hour() > baseTime.Hour() {
		return true
	} else if c.calculatedTime.Minute() > baseTime.Minute() {
		return true
	} else {
		return false
	}
}

func hasPassed(value, tstamp int) bool{
	return value < tstamp
}

func monthHasDom(dom, month, year int) bool {
	switch month{
		case 1:	return dom <= 31
		case 2:
			if isLeapYear(year) {
				return dom <= 29
			} else {
				return dom <= 28
			}
		case 3: return dom <= 31
		case 4: return dom <= 30
		case 5: return dom <= 31
		case 6: return dom <= 30
		case 7: return dom <= 31
		case 8: return dom <= 31
		case 9: return dom <= 30
		case 10: return dom <= 31
		case 11: return dom <= 30
		case 12: return dom <= 31
		default: panic("strange thingys are happening!")
	}
}

// Check if the combination of day(of month), month and year is the weekday dow.
func monthHasDow(dow, dom, month, year int) bool{
	if !monthHasDom(dom, month, year) {return false}
	Nday := dom % 7
	var Nmonth int
	switch month{
		case 1: Nmonth = 0
		case 2: Nmonth = 3
		case 3: Nmonth = 3
		case 4: Nmonth = 6
		case 5: Nmonth = 1
		case 6: Nmonth = 4
		case 7: Nmonth = 6
		case 8: Nmonth = 2
		case 9: Nmonth = 5
		case 10: Nmonth = 0
		case 11: Nmonth = 3
		case 12: Nmonth = 5
	}
	var Nyear int
	temp := year % 100
	if temp != 0{
		Nyear = (temp + (temp / 4)) % 7
	} else {
		Nyear = 0
	}
	Ncent := (3 - ((year / 100) %4)) * 2
	var Nsj int
	if isLeapYear(year) {
		Nsj = -1
	} else {
		Nsj = 0
	}
	W := (Nday + Nmonth + Nyear + Ncent + Nsj) % 7
	return dow == W
}

func isLeapYear(year int) bool{
	return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)
}
//
func setMonth(tstamp time.Time, month int) time.Time {
	if month > 12 || month < 1 { panic("ERROR Month") }
	return tstamp.AddDate(0, -absolute(int(tstamp.Month()), month), 0)
}

func setDay(tstamp time.Time, day int) time.Time {
	if day > 31 || day < 1{ panic("ERROR Day") }
	return tstamp.AddDate(0, 0, -absolute(tstamp.Day(), day))
}

func setHour(tstamp time.Time, hour int) time.Time {
	if hour >= 24 || hour < 0 { panic("ERROR Hour") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Hour(), hour)) * time.Hour)
}

func setMinute(tstamp time.Time, minute int) time.Time {
	if minute >= 60 || minute < 0{ panic("ERROR Minute") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Minute(), minute)) * time.Minute)
}

func setSecond(tstamp time.Time, second int) time.Time {
	if second >= 60 || second < 0 { panic("ERROR Second") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Second(), second)) * time.Second)
}

func setNanoecond(tstamp time.Time, nanosecond int) time.Time {
	return tstamp.Add(time.Duration(-absolute(tstamp.Nanosecond(), nanosecond)) * time.Nanosecond)
}

func absolute(a, b int) int {
	return a - b
}<commit_msg>Added some comments<commit_after>/*
 *        Copyright (C) 2014 Stefan 'glaxx' Luecke
 *
 *        This program is free software: you can redistribute it and/or modify
 *        it under the terms of the GNU Affero General Public License as published
 *        by the Free Software Foundation, either version 3 of the License, or
 *        (at your option) any later version.
 *
 *        This program is distributed in the hope that it will be useful,
 *        but WITHOUT ANY WARRANTY; without even the implied warranty of
 *        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *        GNU Affero General Public License for more details.
 *
 *        You should have received a copy of the GNU Affero General Public License
 *        along with this program.      If not, see <http://www.gnu.org/licenses/>.
 *
 *        Authors:	Stefan Luecke <glaxx@glaxx.net>
 */

 /*
 * TODO List:
 * - Test leap year behavior
 * - Test
 */

// This package allows you to schedule events
package cron

import (
//	"fmt"
	"time"
//	"sort"
	"log"
	"container/list"
)

type crontime struct {
	second []int
	minute []int
	hour   []int
	dow    []int //Day of Week
	dom    []int //Day of Month
	month  []int
	calculatedTime time.Time
	calculationInProgress bool
	eventList list.List
}

// Returns the time.Duration until the next event.
func (c *crontime) DurationUntilNextEvent() time.Duration {
	return c.nextEvent().Sub(time.Now())
}

func (c *crontime) GetNextEvent() time.Time {
	return c.eventList.Front().Value.(time.Time)
}

func (c *crontime) nextEvent() time.Time {
	if !c.calculationInProgress && c.eventList.Len() == 0{
		r := c.calculateEvent(time.Now())
		go c.fillList(r)
		return r
	} else if c.calculationInProgress && c.eventList.Len() == 0{
		// shit just got real aka TODO
		panic("Shit")

	} else if c.eventList.Len() > 0 {
		e := c.eventList.Front()
		r := e.Value.(time.Time)
		c.eventList.Remove(e)
		go c.fillList(c.eventList.Back().Value.(time.Time))
		return r
	}
	panic("shit 2")
}

func (c *crontime) fillList(baseTime time.Time) {
	if c.eventList.Len() == 0 {
		c.eventList.PushBack(c.calculateEvent(baseTime))
	}
	for ; c.eventList.Len() < 5; {
		c.eventList.PushBack(c.calculateEvent(c.eventList.Back().Value.(time.Time)))
	}
}

func (c *crontime) setCalculationInProgress(set bool) {
	c.calculationInProgress = set
}

// This functions calculates the next event
func (c *crontime) calculateEvent(baseTime time.Time) time.Time{
	c.calculationInProgress = true
	defer c.setCalculationInProgress(false)
	baseTime = setNanoecond(baseTime, 10000)
	c.calculatedTime = baseTime // Ignore all Events in the Past & initial 'result'
	//c.calculatedTime = setNanoecond(c.calculatedTime, 10000)
	c.nextValidMonth(baseTime)
	c.nextValidDay(baseTime)
	c.nextValidHour(baseTime)
	c.nextValidMinute(baseTime)
	c.nextValidSecond(baseTime)
	log.Println("Cronbee has found a time stamp: ", c.calculatedTime)
	return c.calculatedTime
}

// Calculates the next valid Month based upon the previous results.
func (c *crontime) nextValidMonth(baseTime time.Time) {
	for _, mon := range c.month {
		if mon >= int(c.calculatedTime.Month()) {
			//log.Print("Inside Month", mon, c.calculatedTime)
			c.calculatedTime = setMonth(c.calculatedTime, mon)
			//log.Println(" :: and out", c.calculatedTime)
			return
		}
	}
	// If no result was found try it again in the following year
	c.calculatedTime = c.calculatedTime.AddDate(1, 0, 0)
	c.calculatedTime = setMonth(c.calculatedTime, c.month[0])
	//log.Println("Cronbee: Month", c.calculatedTime, baseTime, c.month)
	c.nextValidMonth(baseTime)
}

// Calculates the next valid Day based upon the previous results.
func (c *crontime) nextValidDay(baseTime time.Time) {
	for _, dom := range c.dom {
		if dom >= c.calculatedTime.Day() {
			for _, dow := range c.dow {
				if monthHasDow(dow, dom, int(c.calculatedTime.Month()), c.calculatedTime.Year()){
					c.calculatedTime = setDay(c.calculatedTime, dom)
					//log.Println("Cronbee: Day-INS-1:", c.calculatedTime)
					return
				}
			}
		}
	}/* else {
		for _, dow := range c.dow {
			if monthHasDow(dow, dom, int(c.calculatedTime.Month()), c.calculatedTime.Year()){
				c.calculatedTime = setDay(c.calculatedTime, dom)
				log.Println("Cronbee: Day-INS-2:", c.calculatedTime)
				return
			}
		}
	}*/
	// If no result was found try it again in the following month.
	c.calculatedTime = c.calculatedTime.AddDate(0, 1, 0)
	c.calculatedTime = setDay(c.calculatedTime, c.dom[0])
	//log.Println("Cronbee: Day", c.calculatedTime, baseTime)
	c.nextValidMonth(baseTime)
	c.nextValidDay(baseTime)
}

// Calculates the next valid Hour based upon the previous results.
func (c *crontime) nextValidHour(baseTime time.Time) {
	for _, hour := range c.hour {
		if c.calculatedTime.Day() == baseTime.Day() {
			if !hasPassed(hour, c.calculatedTime.Hour()) {
				c.calculatedTime = setHour(c.calculatedTime, hour)
				return
			}
		} else {
			c.calculatedTime = setHour(c.calculatedTime, hour)
			return
		}
	}
	// If no result was found try it again in the following day.
	c.calculatedTime = c.calculatedTime.AddDate(0, 0, 1)     // <-|
	c.calculatedTime = setHour(c.calculatedTime, c.hour[0])  //   |
	//log.Println("Cronbee: Hour", c.calculatedTime, baseTime) // |
	c.nextValidMonth(baseTime) // May trigger a new month --------|
	c.nextValidDay(baseTime)
	c.nextValidHour(baseTime)
}

// Calculates the next valid Minute based upon the previous results.
func (c *crontime) nextValidMinute(baseTime time.Time) {
	for _, min := range c.minute {
		if c.calculatedTime.Hour() == baseTime.Hour() {
			if !hasPassed(min, c.calculatedTime.Minute()) {
				c.calculatedTime = setMinute(c.calculatedTime, min)
				return
			}
		} else {
			c.calculatedTime = setMinute(c.calculatedTime, min)
			return
		}
	}
	c.calculatedTime = c.calculatedTime.Add(1 * time.Hour)
	c.calculatedTime = setMinute(c.calculatedTime, c.minute[0])
	//log.Println("Cronbee: Minute", c.calculatedTime, baseTime)
	c.nextValidHour(baseTime)
	c.nextValidMinute(baseTime)
}

// Calculates the next valid Second based upon the previous results.
func (c *crontime) nextValidSecond(baseTime time.Time) {
	for _, sec := range c.second {
		if !c.minuteHasPassed(baseTime) {
			// check if sec is in the past. <= prevents triggering the same event twice
			if sec > c.calculatedTime.Second() {
				c.calculatedTime = setSecond(c.calculatedTime, sec)
				return
			}
		} else {
			c.calculatedTime = setSecond(c.calculatedTime, sec)
			return
		}
	}
	c.calculatedTime = c.calculatedTime.Add(1 * time.Minute)
	c.calculatedTime = setSecond(c.calculatedTime, 0)
	//log.Println("Cronbee: Second", c.calculatedTime, baseTime)
	c.nextValidMinute(baseTime)
	c.nextValidSecond(baseTime)
}

func (c *crontime) minuteHasPassed(baseTime time.Time) bool {
	if c.calculatedTime.Year() > baseTime.Year() {
		return true
	} else if c.calculatedTime.Month() > baseTime.Month() {
		return true
	} else if c.calculatedTime.Day() > baseTime.Day() {
		return true
	} else if c.calculatedTime.Hour() > baseTime.Hour() {
		return true
	} else if c.calculatedTime.Minute() > baseTime.Minute() {
		return true
	} else {
		return false
	}
}

func hasPassed(value, tstamp int) bool{
	return value < tstamp
}

func monthHasDom(dom, month, year int) bool {
	switch month{
		case 1:	return dom <= 31
		case 2:
			if isLeapYear(year) {
				return dom <= 29
			} else {
				return dom <= 28
			}
		case 3: return dom <= 31
		case 4: return dom <= 30
		case 5: return dom <= 31
		case 6: return dom <= 30
		case 7: return dom <= 31
		case 8: return dom <= 31
		case 9: return dom <= 30
		case 10: return dom <= 31
		case 11: return dom <= 30
		case 12: return dom <= 31
		default: panic("strange thingys are happening!")
	}
}

// Check if the combination of day(of month), month and year is the weekday dow.
func monthHasDow(dow, dom, month, year int) bool{
	if !monthHasDom(dom, month, year) {return false}
	Nday := dom % 7
	var Nmonth int
	switch month{
		case 1: Nmonth = 0
		case 2: Nmonth = 3
		case 3: Nmonth = 3
		case 4: Nmonth = 6
		case 5: Nmonth = 1
		case 6: Nmonth = 4
		case 7: Nmonth = 6
		case 8: Nmonth = 2
		case 9: Nmonth = 5
		case 10: Nmonth = 0
		case 11: Nmonth = 3
		case 12: Nmonth = 5
	}
	var Nyear int
	temp := year % 100
	if temp != 0{
		Nyear = (temp + (temp / 4)) % 7
	} else {
		Nyear = 0
	}
	Ncent := (3 - ((year / 100) %4)) * 2
	var Nsj int
	if isLeapYear(year) {
		Nsj = -1
	} else {
		Nsj = 0
	}
	W := (Nday + Nmonth + Nyear + Ncent + Nsj) % 7
	return dow == W
}

func isLeapYear(year int) bool{
	return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)
}
//
func setMonth(tstamp time.Time, month int) time.Time {
	if month > 12 || month < 1 { panic("ERROR Month") }
	return tstamp.AddDate(0, -absolute(int(tstamp.Month()), month), 0)
}

func setDay(tstamp time.Time, day int) time.Time {
	if day > 31 || day < 1{ panic("ERROR Day") }
	return tstamp.AddDate(0, 0, -absolute(tstamp.Day(), day))
}

func setHour(tstamp time.Time, hour int) time.Time {
	if hour >= 24 || hour < 0 { panic("ERROR Hour") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Hour(), hour)) * time.Hour)
}

func setMinute(tstamp time.Time, minute int) time.Time {
	if minute >= 60 || minute < 0{ panic("ERROR Minute") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Minute(), minute)) * time.Minute)
}

func setSecond(tstamp time.Time, second int) time.Time {
	if second >= 60 || second < 0 { panic("ERROR Second") }
	return tstamp.Add(time.Duration(-absolute(tstamp.Second(), second)) * time.Second)
}

func setNanoecond(tstamp time.Time, nanosecond int) time.Time {
	return tstamp.Add(time.Duration(-absolute(tstamp.Nanosecond(), nanosecond)) * time.Nanosecond)
}

func absolute(a, b int) int {
	return a - b
}<|endoftext|>


<commit_before>package main

import "fmt"

func main() {

}

<commit_msg>Exercise #17<commit_after>package main

import "fmt"

func zeroval(ival int)  {
  ival = 0
}

func zeroptr(iptr *int)  {
  *iptr = 0
}

func main() {
  i := 1
  fmt.Println("initial:",i)

  zeroval(i)
  fmt.Println("zeroval:",i)

  zeroptr(&i)
  fmt.Println("zeroptr:",i)

  fmt.Println("pointer:",&i)
}
<|endoftext|>


<commit_before>package gatherrun

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var (
	abuseInterval     = time.Minute * 15
	analyticsInterval = time.Hour * 24
	envVarName        = "GATHER"
)

type GatherRun struct {
	DestFolder string
	Exporter   Exporter
	Fetcher    Fetcher
	Env        string
	Username   string
	ScriptType string
}

func Run(env, username string) {
	fetcher := &S3Fetcher{
		BucketName: "koding-gather",
		FileName:   "koding-kernel.tar",
		Region:     "us-east-1",
	}

	exporter := NewKodingExporter()

	go func() {
		New(fetcher, exporter, env, username, "abuse").Run()
		New(fetcher, exporter, env, username, "analytics").Run()
	}()

	abuseTimer := time.NewTimer(abuseInterval)
	analyticsTimer := time.NewTimer(analyticsInterval)

	for {
		select {
		case <-abuseTimer.C:
			New(fetcher, exporter, env, username, "abuse").Run()
		case <-analyticsTimer.C:
			New(fetcher, exporter, env, username, "analytics").Run()
		}
	}
}

func New(fetcher Fetcher, exporter Exporter, env, username, scriptType string) *GatherRun {
	tmpDir, err := ioutil.TempDir("/tmp", "koding-kernel")
	if err != nil {
		// TODO: how to deal with errs
	}

	return &GatherRun{
		Fetcher:    fetcher,
		Exporter:   exporter,
		DestFolder: tmpDir,
		Env:        env,
		Username:   username,
		ScriptType: scriptType,
	}
}

func (c *GatherRun) Run() (err error) {
	defer func() { err = c.Cleanup() }()

	binary, err := c.GetGatherBinary()
	if err != nil {
		return err
	}

	if err = c.Export(binary.Run()); err != nil {
		return err
	}

	return nil
}

func (c *GatherRun) GetGatherBinary() (*GatherBinary, error) {
	if err := os.MkdirAll(c.DestFolder, 0777); err != nil {
		return nil, err
	}

	if err := c.DownloadScripts(c.DestFolder); err != nil {
		return nil, err
	}

	tarFile := filepath.Join(c.DestFolder, c.Fetcher.GetFileName())
	if err := untar(tarFile, c.DestFolder); err != nil {
		return nil, err
	}

	binaryPath := strings.TrimSuffix(tarFile, tarSuffix)
	return &GatherBinary{Path: binaryPath, ScriptType: c.ScriptType}, nil
}

func (c *GatherRun) DownloadScripts(folderName string) error {
	return c.Fetcher.Download(folderName)
}

func (c *GatherRun) Export(raw []interface{}, err error) error {
	if err != nil {
		return c.sendErrors(err)
	}

	var stats = []GatherSingleStat{}
	var errors = []error{}

	for _, r := range raw {
		buf := new(bytes.Buffer)
		if err := json.NewEncoder(buf).Encode(r); err != nil {
			errors = append(errors, err)
			continue
		}

		var stat GatherSingleStat
		if err := json.NewDecoder(buf).Decode(&stat); err != nil {
			errors = append(errors, err)
			continue
		}

		stats = append(stats, stat)
	}

	if len(errors) > 0 {
		c.sendErrors(errors...)
	}

	if len(stats) > 0 {
		gStat := &GatherStat{
			Env: c.Env, Username: c.Username, Stats: stats, Type: c.ScriptType,
		}
		return c.Exporter.SendStats(gStat)
	}

	return nil
}

func (c *GatherRun) Cleanup() error {
	return os.RemoveAll(c.DestFolder)
}

func (c *GatherRun) sendErrors(errs ...error) error {
	gErr := &GatherError{
		Env: c.Env, Username: c.Username, Errors: errs,
	}

	return c.Exporter.SendError(gErr)
}
<commit_msg>gather: remove running when klient starts<commit_after>package gatherrun

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

var (
	abuseInterval     = time.Minute * 15
	analyticsInterval = time.Hour * 24
	envVarName        = "GATHER"
)

type GatherRun struct {
	DestFolder string
	Exporter   Exporter
	Fetcher    Fetcher
	Env        string
	Username   string
	ScriptType string
}

func Run(env, username string) {
	fetcher := &S3Fetcher{
		BucketName: "koding-gather",
		FileName:   "koding-kernel.tar",
		Region:     "us-east-1",
	}

	exporter := NewKodingExporter()

	abuseTimer := time.NewTimer(abuseInterval)
	analyticsTimer := time.NewTimer(analyticsInterval)

	for {
		select {
		case <-abuseTimer.C:
			New(fetcher, exporter, env, username, "abuse").Run()
		case <-analyticsTimer.C:
			New(fetcher, exporter, env, username, "analytics").Run()
		}
	}
}

func New(fetcher Fetcher, exporter Exporter, env, username, scriptType string) *GatherRun {
	tmpDir, err := ioutil.TempDir("/tmp", "koding-kernel")
	if err != nil {
		// TODO: how to deal with errs
	}

	return &GatherRun{
		Fetcher:    fetcher,
		Exporter:   exporter,
		DestFolder: tmpDir,
		Env:        env,
		Username:   username,
		ScriptType: scriptType,
	}
}

func (c *GatherRun) Run() (err error) {
	defer func() { err = c.Cleanup() }()

	binary, err := c.GetGatherBinary()
	if err != nil {
		return err
	}

	if err = c.Export(binary.Run()); err != nil {
		return err
	}

	return nil
}

func (c *GatherRun) GetGatherBinary() (*GatherBinary, error) {
	if err := os.MkdirAll(c.DestFolder, 0777); err != nil {
		return nil, err
	}

	if err := c.DownloadScripts(c.DestFolder); err != nil {
		return nil, err
	}

	tarFile := filepath.Join(c.DestFolder, c.Fetcher.GetFileName())
	if err := untar(tarFile, c.DestFolder); err != nil {
		return nil, err
	}

	binaryPath := strings.TrimSuffix(tarFile, tarSuffix)
	return &GatherBinary{Path: binaryPath, ScriptType: c.ScriptType}, nil
}

func (c *GatherRun) DownloadScripts(folderName string) error {
	return c.Fetcher.Download(folderName)
}

func (c *GatherRun) Export(raw []interface{}, err error) error {
	if err != nil {
		return c.sendErrors(err)
	}

	var stats = []GatherSingleStat{}
	var errors = []error{}

	for _, r := range raw {
		buf := new(bytes.Buffer)
		if err := json.NewEncoder(buf).Encode(r); err != nil {
			errors = append(errors, err)
			continue
		}

		var stat GatherSingleStat
		if err := json.NewDecoder(buf).Decode(&stat); err != nil {
			errors = append(errors, err)
			continue
		}

		stats = append(stats, stat)
	}

	if len(errors) > 0 {
		c.sendErrors(errors...)
	}

	if len(stats) > 0 {
		gStat := &GatherStat{
			Env: c.Env, Username: c.Username, Stats: stats, Type: c.ScriptType,
		}
		return c.Exporter.SendStats(gStat)
	}

	return nil
}

func (c *GatherRun) Cleanup() error {
	return os.RemoveAll(c.DestFolder)
}

func (c *GatherRun) sendErrors(errs ...error) error {
	gErr := &GatherError{
		Env: c.Env, Username: c.Username, Errors: errs,
	}

	return c.Exporter.SendError(gErr)
}
<|endoftext|>


<commit_before>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package linuxapparmorprofile

import (
	"errors"
	"github.com/huawei-openlab/oct/tools/specsValidator/adaptor"
	"github.com/huawei-openlab/oct/tools/specsValidator/manager"
	"github.com/huawei-openlab/oct/tools/specsValidator/utils/configconvert"
	"github.com/opencontainers/specs"
	"log"
	"os/exec"
	"runtime"
	"strings"
)

/**
*Need mount proc and set mnt namespace when get namespace from container
*and the specs.Process.Terminal must be false when call runc in programe.
 */
var linuxSpec specs.LinuxSpec = specs.LinuxSpec{
	Spec: specs.Spec{
		Version: "pre-draft",
		Platform: specs.Platform{
			OS:   runtime.GOOS,
			Arch: runtime.GOARCH,
		},
		Root: specs.Root{
			Path:     "rootfs_rootconfig",
			Readonly: true,
		},
		Process: specs.Process{
			Terminal: false,
			User: specs.User{
				UID:            0,
				GID:            0,
				AdditionalGids: nil,
			},
			Args: []string{""},
		},
		Mounts: []specs.Mount{
			{
				Type:        "proc",
				Source:      "proc",
				Destination: "/proc",
				Options:     "",
			},
		},
	},
	Linux: specs.Linux{
		Resources: specs.Resources{
			Memory: specs.Memory{
				Swappiness: -1,
			},
		},
		Namespaces: []specs.Namespace{
			{
				Type: "mount",
				Path: "",
			},
		},
	},
}

var TestSuiteLinuxApparmorProfile manager.TestSuite = manager.TestSuite{Name: "LinuxSpec.Linux.ApparmorProfile"}

func init() {
	TestSuiteLinuxApparmorProfile.AddTestCase("TestLinuxApparmorProfile", TestLinuxApparmorProfile)
}

func setApparmorProfile(profilename string) specs.LinuxSpec {
	linuxSpec.Linux.ApparmorProfile = profilename
	linuxSpec.Spec.Process.Args = []string{"/bin/bash", "-c", "echo \" enter the container \""}
	return linuxSpec
}

func testApparmorProfile(linuxSpec *specs.LinuxSpec) (string, error) {
	configFile := "./config.json"
	inputprofilename := linuxSpec.Linux.ApparmorProfile
	err := configconvert.LinuxSpecToConfig(configFile, linuxSpec)
	cmd := exec.Command("bash", "-c", "cp cases/linuxapparmorprofile/"+inputprofilename+" /etc/apparmor.d")
	_, err = cmd.Output()
	if err != nil {
		log.Fatalf("[specsValidator] linux apparmor profile test : copy the apparmor file error, %v", err)
	}
	out, err := adaptor.StartRunc(configFile)
	if err != nil {
		return manager.UNKNOWNERR, errors.New(out + err.Error())
	}
	cmd = exec.Command("bash", "-c", " apparmor_status")
	cmdout, err := cmd.Output()
	if err != nil {
		return manager.UNSPPORTED, errors.New("HOST Machine NOT Support Apparmor")
	} else if strings.Contains(strings.TrimSpace(string(cmdout)), inputprofilename) {
		return manager.PASSED, nil
	} else {
		return manager.FAILED, nil
	}
}
<commit_msg>modify linuxapparmor test case<commit_after>//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package linuxapparmorprofile

import (
	"errors"
	"github.com/huawei-openlab/oct/tools/specsValidator/adaptor"
	"github.com/huawei-openlab/oct/tools/specsValidator/manager"
	"github.com/huawei-openlab/oct/tools/specsValidator/utils/configconvert"
	"github.com/opencontainers/specs"
	"log"
	"os/exec"
	"runtime"
	"strings"
)

/**
*Need mount proc and set mnt namespace when get namespace from container
*and the specs.Process.Terminal must be false when call runc in programe.
 */
var linuxSpec specs.LinuxSpec = specs.LinuxSpec{
	Spec: specs.Spec{
		Version: "pre-draft",
		Platform: specs.Platform{
			OS:   runtime.GOOS,
			Arch: runtime.GOARCH,
		},
		Root: specs.Root{
			Path:     "rootfs_rootconfig",
			Readonly: true,
		},
		Process: specs.Process{
			Terminal: false,
			User: specs.User{
				UID:            0,
				GID:            0,
				AdditionalGids: nil,
			},
			Args: []string{""},
		},
		Mounts: []specs.Mount{
			{
				Type:        "proc",
				Source:      "proc",
				Destination: "/proc",
				Options:     "",
			},
		},
	},
	Linux: specs.Linux{
		Resources: specs.Resources{
			Memory: specs.Memory{
				Swappiness: -1,
			},
		},
		Namespaces: []specs.Namespace{
			{
				Type: "mount",
				Path: "",
			},
		},
	},
}

var TestSuiteLinuxApparmorProfile manager.TestSuite = manager.TestSuite{Name: "LinuxSpec.Linux.ApparmorProfile"}

func init() {
	TestSuiteLinuxApparmorProfile.AddTestCase("TestLinuxApparmorProfile", TestLinuxApparmorProfile)
	manager.Manager.AddTestSuite(TestSuiteLinuxApparmorProfile)
}

func setApparmorProfile(profilename string) specs.LinuxSpec {
	linuxSpec.Linux.ApparmorProfile = profilename
	linuxSpec.Spec.Process.Args = []string{"/bin/bash", "-c", "echo \" enter the container \""}
	return linuxSpec
}

func testApparmorProfile(linuxSpec *specs.LinuxSpec) (string, error) {
	configFile := "./config.json"
	inputprofilename := linuxSpec.Linux.ApparmorProfile
	err := configconvert.LinuxSpecToConfig(configFile, linuxSpec)
	cmd := exec.Command("bash", "-c", "cp cases/linuxapparmorprofile/"+inputprofilename+" /etc/apparmor.d")
	_, err = cmd.Output()
	if err != nil {
		log.Fatalf("[specsValidator] linux apparmor profile test : copy the apparmor file error, %v", err)
	}
	out, err := adaptor.StartRunc(configFile)
	if err != nil {
		return manager.UNKNOWNERR, errors.New(out + err.Error())
	}
	cmd = exec.Command("bash", "-c", " apparmor_status")
	cmdout, err := cmd.Output()
	if err != nil {
		return manager.UNSPPORTED, errors.New("HOST Machine NOT Support Apparmor")
	} else if strings.Contains(strings.TrimSpace(string(cmdout)), inputprofilename) {
		return manager.PASSED, nil
	} else {
		return manager.FAILED, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/keyman"
	"github.com/getlantern/tlsdialer"

	"github.com/getlantern/flashlight/client"
)

const (
	numberOfWorkers = 50
	ftVersionFile   = `https://raw.githubusercontent.com/firetweet/downloads/master/version.txt`
)

var (
	help            = flag.Bool("help", false, "Get usage help")
	domainsFile     = flag.String("domains", "", "Path to file containing list of domains to use, with one domain per line (e.g. domains.txt)")
	blacklistFile   = flag.String("blacklist", "", "Path to file containing list of blacklisted domains, which will be excluded from the configuration even if present in the domains file (e.g. blacklist.txt)")
	proxiedSitesDir = flag.String("proxiedsites", "proxiedsites", "Path to directory containing proxied site lists, which will be combined and proxied by Lantern")
	minFreq         = flag.Float64("minfreq", 3.0, "Minimum frequency (percentage) for including CA cert in list of trusted certs, defaults to 3.0%")

	// Note - you can get the content for the fallbacksFile from https://lanternctrl1-2.appspot.com/listfallbacks
	fallbacksFile = flag.String("fallbacks", "fallbacks.json", "File containing json array of fallback information")
)

var (
	log = golog.LoggerFor("genconfig")

	domains []string

	blacklist    = make(filter)
	proxiedSites = make(filter)
	fallbacks    []map[string]interface{}
	ftVersion    string

	domainsCh     = make(chan string)
	masqueradesCh = make(chan *masquerade)
	wg            sync.WaitGroup
)

type filter map[string]bool

type masquerade struct {
	Domain    string
	IpAddress string
	RootCA    *castat
}

type castat struct {
	CommonName string
	Cert       string
	freq       float64
}

func main() {
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	numcores := runtime.NumCPU()
	log.Debugf("Using all %d cores on machine", numcores)
	runtime.GOMAXPROCS(numcores)

	loadDomains()
	loadProxiedSitesList()
	loadBlacklist()
	loadFallbacks()
	loadFtVersion()

	masqueradesTmpl := loadTemplate("masquerades.go.tmpl")
	proxiedSitesTmpl := loadTemplate("proxiedsites.go.tmpl")
	fallbacksTmpl := loadTemplate("fallbacks.go.tmpl")
	yamlTmpl := loadTemplate("cloud.yaml.tmpl")

	go feedDomains()
	cas, masquerades := coalesceMasquerades()
	model := buildModel(cas, masquerades)
	generateTemplate(model, yamlTmpl, "cloud.yaml")
	generateTemplate(model, masqueradesTmpl, "../config/masquerades.go")
	_, err := run("gofmt", "-w", "../config/masquerades.go")
	if err != nil {
		log.Fatalf("Unable to format masquerades.go: %s", err)
	}
	generateTemplate(model, proxiedSitesTmpl, "../config/proxiedsites.go")
	_, err = run("gofmt", "-w", "../config/proxiedsites.go")
	if err != nil {
		log.Fatalf("Unable to format proxiedsites.go: %s", err)
	}
	generateTemplate(model, fallbacksTmpl, "../config/fallbacks.go")
	_, err = run("gofmt", "-w", "../config/fallbacks.go")
	if err != nil {
		log.Fatalf("Unable to format fallbacks.go: %s", err)
	}
}

func loadDomains() {
	if *domainsFile == "" {
		log.Error("Please specify a domains file")
		flag.Usage()
		os.Exit(2)
	}
	domainsBytes, err := ioutil.ReadFile(*domainsFile)
	if err != nil {
		log.Fatalf("Unable to read domains file at %s: %s", *domainsFile, err)
	}
	domains = strings.Split(string(domainsBytes), "\n")
}

// Scans the proxied site directory and stores the sites in the files found
func loadProxiedSites(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		// skip root directory
		return nil
	}
	proxiedSiteBytes, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatalf("Unable to read blacklist file at %s: %s", path, err)
	}
	for _, domain := range strings.Split(string(proxiedSiteBytes), "\n") {
		// skip empty lines, comments, and *.ir sites
		// since we're focusing on Iran with this first release, we aren't adding *.ir sites
		// to the global proxied sites
		// to avoid proxying sites that are already unblocked there.
		// This is a general problem when you aren't maintaining country-specific whitelists
		// which will be addressed in the next phase
		if domain != "" && !strings.HasPrefix(domain, "#") && !strings.HasSuffix(domain, ".ir") {
			proxiedSites[domain] = true
		}
	}
	return err
}

func loadProxiedSitesList() {
	if *proxiedSitesDir == "" {
		log.Error("Please specify a proxied site directory")
		flag.Usage()
		os.Exit(3)
	}

	err := filepath.Walk(*proxiedSitesDir, loadProxiedSites)
	if err != nil {
		log.Errorf("Could not open proxied site directory: %s", err)
	}
}

func loadFtVersion() {
	res, err := http.Get(ftVersionFile)
	if err != nil {
		log.Fatalf("Error fetching FireTweet version file: %s", err)
	}

	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Could not read FT version file: %s", err)
	}
	ftVersion = strings.TrimSpace(string(body))
}

func loadBlacklist() {
	if *blacklistFile == "" {
		log.Error("Please specify a blacklist file")
		flag.Usage()
		os.Exit(3)
	}
	blacklistBytes, err := ioutil.ReadFile(*blacklistFile)
	if err != nil {
		log.Fatalf("Unable to read blacklist file at %s: %s", *blacklistFile, err)
	}
	for _, domain := range strings.Split(string(blacklistBytes), "\n") {
		blacklist[domain] = true
	}
}

func loadFallbacks() {
	if *fallbacksFile == "" {
		log.Error("Please specify a fallbacks file")
		flag.Usage()
		os.Exit(2)
	}
	fallbacksBytes, err := ioutil.ReadFile(*fallbacksFile)
	if err != nil {
		log.Fatalf("Unable to read fallbacks file at %s: %s", *fallbacksFile, err)
	}
	err = json.Unmarshal(fallbacksBytes, &fallbacks)
	if err != nil {
		log.Fatalf("Unable to unmarshal json from %v: %v", *fallbacksFile, err)
	}
}

func loadTemplate(name string) string {
	bytes, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatalf("Unable to load template %s: %s", name, err)
	}
	return string(bytes)
}

func feedDomains() {
	wg.Add(numberOfWorkers)
	for i := 0; i < numberOfWorkers; i++ {
		go grabCerts()
	}

	for _, domain := range domains {
		domainsCh <- domain
	}
	close(domainsCh)
	wg.Wait()
	close(masqueradesCh)
}

// grabCerts grabs certificates for the domains received on domainsCh and sends
// *masquerades to masqueradesCh.
func grabCerts() {
	defer wg.Done()

	for domain := range domainsCh {
		_, blacklisted := blacklist[domain]
		if blacklisted {
			log.Tracef("Domain %s is blacklisted, skipping", domain)
			continue
		}
		log.Tracef("Grabbing certs for domain: %s", domain)
		cwt, err := tlsdialer.DialForTimings(&net.Dialer{
			Timeout: 10 * time.Second,
		}, "tcp", domain+":443", false, nil)
		if err != nil {
			log.Errorf("Unable to dial domain %s: %s", domain, err)
			continue
		}
		cwt.Conn.Close()
		chain := cwt.VerifiedChains[0]
		rootCA := chain[len(chain)-1]
		rootCert, err := keyman.LoadCertificateFromX509(rootCA)
		if err != nil {
			log.Errorf("Unablet to load keyman certificate: %s", err)
			continue
		}
		ca := &castat{
			CommonName: rootCA.Subject.CommonName,
			Cert:       strings.Replace(string(rootCert.PEMEncoded()), "\n", "\\n", -1),
		}
		masqueradesCh <- &masquerade{
			Domain:    domain,
			IpAddress: cwt.ResolvedAddr.IP.String(),
			RootCA:    ca,
		}
	}
}

func coalesceMasquerades() (map[string]*castat, []*masquerade) {
	count := 0
	allCAs := make(map[string]*castat)
	allMasquerades := make([]*masquerade, 0)
	for masquerade := range masqueradesCh {
		count = count + 1
		ca := allCAs[masquerade.RootCA.Cert]
		if ca == nil {
			ca = masquerade.RootCA
		}
		ca.freq = ca.freq + 1
		allCAs[ca.Cert] = ca
		allMasquerades = append(allMasquerades, masquerade)
	}

	// Trust only those cas whose relative frequency exceeds *minFreq
	trustedCAs := make(map[string]*castat)
	for _, ca := range allCAs {
		// Make frequency relative
		ca.freq = float64(ca.freq*100) / float64(count)
		if ca.freq > *minFreq {
			trustedCAs[ca.Cert] = ca
		}
	}

	// Pick only the masquerades associated with the trusted certs
	trustedMasquerades := make([]*masquerade, 0)
	for _, masquerade := range allMasquerades {
		_, caFound := trustedCAs[masquerade.RootCA.Cert]
		if caFound {
			trustedMasquerades = append(trustedMasquerades, masquerade)
		}
	}

	return trustedCAs, trustedMasquerades
}

func buildModel(cas map[string]*castat, masquerades []*masquerade) map[string]interface{} {
	casList := make([]*castat, 0, len(cas))
	for _, ca := range cas {
		casList = append(casList, ca)
	}
	sort.Sort(ByFreq(casList))
	sort.Sort(ByDomain(masquerades))
	ps := make([]string, 0, len(proxiedSites))
	for site, _ := range proxiedSites {
		ps = append(ps, site)
	}
	sort.Strings(ps)
	fbs := make([]map[string]interface{}, 0, len(fallbacks))
	for _, fb := range fallbacks {
		ip := fb["ip"].(string)
		if fb["pt"] != nil {
			log.Debugf("Skipping fallback %v because it has pluggable transport enabled", ip)
			continue
		}

		cert := fb["cert"].(string)
		// Replace newlines in cert with newline literals
		fb["cert"] = strings.Replace(cert, "\n", "\\n", -1)

		// Test connectivity
		info := &client.ChainedServerInfo{
			Addr:      ip + ":443",
			Cert:      cert,
			AuthToken: fb["auth_token"].(string),
			Pipelined: true,
		}
		dialer, err := info.Dialer()
		if err != nil {
			log.Debugf("Skipping fallback %v because of error building dialer: %v", ip, err)
			continue
		}
		conn, err := dialer.Dial("tcp", "http://www.google.com")
		if err != nil {
			log.Debugf("Skipping fallback %v because dialing Google failed: %v", ip, err)
			continue
		}
		conn.Close()

		// Use this fallback
		fbs = append(fbs, fb)
	}
	return map[string]interface{}{
		"cas":          casList,
		"masquerades":  masquerades,
		"proxiedsites": ps,
		"fallbacks":    fbs,
		"ftVersion":    ftVersion,
	}
}

func generateTemplate(model map[string]interface{}, tmplString string, filename string) {
	tmpl, err := template.New(filename).Funcs(funcMap).Parse(tmplString)
	if err != nil {
		log.Errorf("Unable to parse template: %s", err)
		return
	}
	out, err := os.Create(filename)
	if err != nil {
		log.Errorf("Unable to create %s: %s", filename, err)
		return
	}
	defer out.Close()
	err = tmpl.Execute(out, model)
	if err != nil {
		log.Errorf("Unable to generate %s: %s", filename, err)
	}
}

func run(prg string, args ...string) (string, error) {
	cmd := exec.Command(prg, args...)
	log.Debugf("Running %s %s", prg, strings.Join(args, " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%s says %s", prg, string(out))
	}
	return string(out), nil
}

func base64Encode(sites []string) string {
	raw, err := json.Marshal(sites)
	if err != nil {
		panic(fmt.Errorf("Unable to marshal proxied sites: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(raw)
	return b64
}

// the functions to be called from template
var funcMap = template.FuncMap{
	"encode": base64Encode,
}

type ByDomain []*masquerade

func (a ByDomain) Len() int           { return len(a) }
func (a ByDomain) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDomain) Less(i, j int) bool { return a[i].Domain < a[j].Domain }

type ByFreq []*castat

func (a ByFreq) Len() int           { return len(a) }
func (a ByFreq) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFreq) Less(i, j int) bool { return a[i].freq > a[j].freq }
<commit_msg>Errcheck'ed github.com/getlantern/flashlight/genconfig<commit_after>package main

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/keyman"
	"github.com/getlantern/tlsdialer"

	"github.com/getlantern/flashlight/client"
)

const (
	numberOfWorkers = 50
	ftVersionFile   = `https://raw.githubusercontent.com/firetweet/downloads/master/version.txt`
)

var (
	help            = flag.Bool("help", false, "Get usage help")
	domainsFile     = flag.String("domains", "", "Path to file containing list of domains to use, with one domain per line (e.g. domains.txt)")
	blacklistFile   = flag.String("blacklist", "", "Path to file containing list of blacklisted domains, which will be excluded from the configuration even if present in the domains file (e.g. blacklist.txt)")
	proxiedSitesDir = flag.String("proxiedsites", "proxiedsites", "Path to directory containing proxied site lists, which will be combined and proxied by Lantern")
	minFreq         = flag.Float64("minfreq", 3.0, "Minimum frequency (percentage) for including CA cert in list of trusted certs, defaults to 3.0%")

	// Note - you can get the content for the fallbacksFile from https://lanternctrl1-2.appspot.com/listfallbacks
	fallbacksFile = flag.String("fallbacks", "fallbacks.json", "File containing json array of fallback information")
)

var (
	log = golog.LoggerFor("genconfig")

	domains []string

	blacklist    = make(filter)
	proxiedSites = make(filter)
	fallbacks    []map[string]interface{}
	ftVersion    string

	domainsCh     = make(chan string)
	masqueradesCh = make(chan *masquerade)
	wg            sync.WaitGroup
)

type filter map[string]bool

type masquerade struct {
	Domain    string
	IpAddress string
	RootCA    *castat
}

type castat struct {
	CommonName string
	Cert       string
	freq       float64
}

func main() {
	flag.Parse()

	if *help {
		flag.Usage()
		os.Exit(1)
	}

	numcores := runtime.NumCPU()
	log.Debugf("Using all %d cores on machine", numcores)
	runtime.GOMAXPROCS(numcores)

	loadDomains()
	loadProxiedSitesList()
	loadBlacklist()
	loadFallbacks()
	loadFtVersion()

	masqueradesTmpl := loadTemplate("masquerades.go.tmpl")
	proxiedSitesTmpl := loadTemplate("proxiedsites.go.tmpl")
	fallbacksTmpl := loadTemplate("fallbacks.go.tmpl")
	yamlTmpl := loadTemplate("cloud.yaml.tmpl")

	go feedDomains()
	cas, masquerades := coalesceMasquerades()
	model := buildModel(cas, masquerades)
	generateTemplate(model, yamlTmpl, "cloud.yaml")
	generateTemplate(model, masqueradesTmpl, "../config/masquerades.go")
	_, err := run("gofmt", "-w", "../config/masquerades.go")
	if err != nil {
		log.Fatalf("Unable to format masquerades.go: %s", err)
	}
	generateTemplate(model, proxiedSitesTmpl, "../config/proxiedsites.go")
	_, err = run("gofmt", "-w", "../config/proxiedsites.go")
	if err != nil {
		log.Fatalf("Unable to format proxiedsites.go: %s", err)
	}
	generateTemplate(model, fallbacksTmpl, "../config/fallbacks.go")
	_, err = run("gofmt", "-w", "../config/fallbacks.go")
	if err != nil {
		log.Fatalf("Unable to format fallbacks.go: %s", err)
	}
}

func loadDomains() {
	if *domainsFile == "" {
		log.Error("Please specify a domains file")
		flag.Usage()
		os.Exit(2)
	}
	domainsBytes, err := ioutil.ReadFile(*domainsFile)
	if err != nil {
		log.Fatalf("Unable to read domains file at %s: %s", *domainsFile, err)
	}
	domains = strings.Split(string(domainsBytes), "\n")
}

// Scans the proxied site directory and stores the sites in the files found
func loadProxiedSites(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		// skip root directory
		return nil
	}
	proxiedSiteBytes, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatalf("Unable to read blacklist file at %s: %s", path, err)
	}
	for _, domain := range strings.Split(string(proxiedSiteBytes), "\n") {
		// skip empty lines, comments, and *.ir sites
		// since we're focusing on Iran with this first release, we aren't adding *.ir sites
		// to the global proxied sites
		// to avoid proxying sites that are already unblocked there.
		// This is a general problem when you aren't maintaining country-specific whitelists
		// which will be addressed in the next phase
		if domain != "" && !strings.HasPrefix(domain, "#") && !strings.HasSuffix(domain, ".ir") {
			proxiedSites[domain] = true
		}
	}
	return err
}

func loadProxiedSitesList() {
	if *proxiedSitesDir == "" {
		log.Error("Please specify a proxied site directory")
		flag.Usage()
		os.Exit(3)
	}

	err := filepath.Walk(*proxiedSitesDir, loadProxiedSites)
	if err != nil {
		log.Errorf("Could not open proxied site directory: %s", err)
	}
}

func loadFtVersion() {
	res, err := http.Get(ftVersionFile)
	if err != nil {
		log.Fatalf("Error fetching FireTweet version file: %s", err)
	}

	defer func() {
		if err := res.Body.Close(); err != nil {
			log.Debugf("Error closing response body: %v", err)
		}
	}()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatalf("Could not read FT version file: %s", err)
	}
	ftVersion = strings.TrimSpace(string(body))
}

func loadBlacklist() {
	if *blacklistFile == "" {
		log.Error("Please specify a blacklist file")
		flag.Usage()
		os.Exit(3)
	}
	blacklistBytes, err := ioutil.ReadFile(*blacklistFile)
	if err != nil {
		log.Fatalf("Unable to read blacklist file at %s: %s", *blacklistFile, err)
	}
	for _, domain := range strings.Split(string(blacklistBytes), "\n") {
		blacklist[domain] = true
	}
}

func loadFallbacks() {
	if *fallbacksFile == "" {
		log.Error("Please specify a fallbacks file")
		flag.Usage()
		os.Exit(2)
	}
	fallbacksBytes, err := ioutil.ReadFile(*fallbacksFile)
	if err != nil {
		log.Fatalf("Unable to read fallbacks file at %s: %s", *fallbacksFile, err)
	}
	err = json.Unmarshal(fallbacksBytes, &fallbacks)
	if err != nil {
		log.Fatalf("Unable to unmarshal json from %v: %v", *fallbacksFile, err)
	}
}

func loadTemplate(name string) string {
	bytes, err := ioutil.ReadFile(name)
	if err != nil {
		log.Fatalf("Unable to load template %s: %s", name, err)
	}
	return string(bytes)
}

func feedDomains() {
	wg.Add(numberOfWorkers)
	for i := 0; i < numberOfWorkers; i++ {
		go grabCerts()
	}

	for _, domain := range domains {
		domainsCh <- domain
	}
	close(domainsCh)
	wg.Wait()
	close(masqueradesCh)
}

// grabCerts grabs certificates for the domains received on domainsCh and sends
// *masquerades to masqueradesCh.
func grabCerts() {
	defer wg.Done()

	for domain := range domainsCh {
		_, blacklisted := blacklist[domain]
		if blacklisted {
			log.Tracef("Domain %s is blacklisted, skipping", domain)
			continue
		}
		log.Tracef("Grabbing certs for domain: %s", domain)
		cwt, err := tlsdialer.DialForTimings(&net.Dialer{
			Timeout: 10 * time.Second,
		}, "tcp", domain+":443", false, nil)
		if err != nil {
			log.Errorf("Unable to dial domain %s: %s", domain, err)
			continue
		}
		if err := cwt.Conn.Close(); err != nil {
			log.Debugf("Error closing connection: %v", err)
		}
		chain := cwt.VerifiedChains[0]
		rootCA := chain[len(chain)-1]
		rootCert, err := keyman.LoadCertificateFromX509(rootCA)
		if err != nil {
			log.Errorf("Unablet to load keyman certificate: %s", err)
			continue
		}
		ca := &castat{
			CommonName: rootCA.Subject.CommonName,
			Cert:       strings.Replace(string(rootCert.PEMEncoded()), "\n", "\\n", -1),
		}
		masqueradesCh <- &masquerade{
			Domain:    domain,
			IpAddress: cwt.ResolvedAddr.IP.String(),
			RootCA:    ca,
		}
	}
}

func coalesceMasquerades() (map[string]*castat, []*masquerade) {
	count := 0
	allCAs := make(map[string]*castat)
	allMasquerades := make([]*masquerade, 0)
	for masquerade := range masqueradesCh {
		count = count + 1
		ca := allCAs[masquerade.RootCA.Cert]
		if ca == nil {
			ca = masquerade.RootCA
		}
		ca.freq = ca.freq + 1
		allCAs[ca.Cert] = ca
		allMasquerades = append(allMasquerades, masquerade)
	}

	// Trust only those cas whose relative frequency exceeds *minFreq
	trustedCAs := make(map[string]*castat)
	for _, ca := range allCAs {
		// Make frequency relative
		ca.freq = float64(ca.freq*100) / float64(count)
		if ca.freq > *minFreq {
			trustedCAs[ca.Cert] = ca
		}
	}

	// Pick only the masquerades associated with the trusted certs
	trustedMasquerades := make([]*masquerade, 0)
	for _, masquerade := range allMasquerades {
		_, caFound := trustedCAs[masquerade.RootCA.Cert]
		if caFound {
			trustedMasquerades = append(trustedMasquerades, masquerade)
		}
	}

	return trustedCAs, trustedMasquerades
}

func buildModel(cas map[string]*castat, masquerades []*masquerade) map[string]interface{} {
	casList := make([]*castat, 0, len(cas))
	for _, ca := range cas {
		casList = append(casList, ca)
	}
	sort.Sort(ByFreq(casList))
	sort.Sort(ByDomain(masquerades))
	ps := make([]string, 0, len(proxiedSites))
	for site, _ := range proxiedSites {
		ps = append(ps, site)
	}
	sort.Strings(ps)
	fbs := make([]map[string]interface{}, 0, len(fallbacks))
	for _, fb := range fallbacks {
		ip := fb["ip"].(string)
		if fb["pt"] != nil {
			log.Debugf("Skipping fallback %v because it has pluggable transport enabled", ip)
			continue
		}

		cert := fb["cert"].(string)
		// Replace newlines in cert with newline literals
		fb["cert"] = strings.Replace(cert, "\n", "\\n", -1)

		// Test connectivity
		info := &client.ChainedServerInfo{
			Addr:      ip + ":443",
			Cert:      cert,
			AuthToken: fb["auth_token"].(string),
			Pipelined: true,
		}
		dialer, err := info.Dialer()
		if err != nil {
			log.Debugf("Skipping fallback %v because of error building dialer: %v", ip, err)
			continue
		}
		conn, err := dialer.Dial("tcp", "http://www.google.com")
		if err != nil {
			log.Debugf("Skipping fallback %v because dialing Google failed: %v", ip, err)
			continue
		}
		if err := conn.Close(); err != nil {
			log.Debugf("Error closing connection: %v", err)
		}

		// Use this fallback
		fbs = append(fbs, fb)
	}
	return map[string]interface{}{
		"cas":          casList,
		"masquerades":  masquerades,
		"proxiedsites": ps,
		"fallbacks":    fbs,
		"ftVersion":    ftVersion,
	}
}

func generateTemplate(model map[string]interface{}, tmplString string, filename string) {
	tmpl, err := template.New(filename).Funcs(funcMap).Parse(tmplString)
	if err != nil {
		log.Errorf("Unable to parse template: %s", err)
		return
	}
	out, err := os.Create(filename)
	if err != nil {
		log.Errorf("Unable to create %s: %s", filename, err)
		return
	}
	defer func() {
		if err := out.Close(); err != nil {
			log.Debugf("Error closing file: %v", err)
		}
	}()
	err = tmpl.Execute(out, model)
	if err != nil {
		log.Errorf("Unable to generate %s: %s", filename, err)
	}
}

func run(prg string, args ...string) (string, error) {
	cmd := exec.Command(prg, args...)
	log.Debugf("Running %s %s", prg, strings.Join(args, " "))
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%s says %s", prg, string(out))
	}
	return string(out), nil
}

func base64Encode(sites []string) string {
	raw, err := json.Marshal(sites)
	if err != nil {
		panic(fmt.Errorf("Unable to marshal proxied sites: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(raw)
	return b64
}

// the functions to be called from template
var funcMap = template.FuncMap{
	"encode": base64Encode,
}

type ByDomain []*masquerade

func (a ByDomain) Len() int           { return len(a) }
func (a ByDomain) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDomain) Less(i, j int) bool { return a[i].Domain < a[j].Domain }

type ByFreq []*castat

func (a ByFreq) Len() int           { return len(a) }
func (a ByFreq) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFreq) Less(i, j int) bool { return a[i].freq > a[j].freq }
<|endoftext|>


<commit_before>package generator

import (
	"bytes"
	"fmt"
	"html/template"
	"os"

	"github.com/gulien/orbit/context"
)

type (
	// OrbitGenerator struct helps to generate a file from a template.
	OrbitGenerator struct {
		// context is an instance of OrbitContext.
		context *context.OrbitContext
	}
)

// NewOrbitGenerator func instantiates a new instance of OrbitGenerator.
func NewOrbitGenerator(context *context.OrbitContext) *OrbitGenerator {
	return &OrbitGenerator{
		context,
	}
}

// Parse function parses a template and populates it.
func (g *OrbitGenerator) Parse() (bytes.Buffer, error) {
	tmpl := template.New(g.context.TemplateFilePath)
	tmpl, err := tmpl.ParseFiles(g.context.TemplateFilePath)

	if err != nil {
		return nil, fmt.Errorf("Failed to parse the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	var data bytes.Buffer
	if err := tmpl.Execute(&data, g.context); err != nil {
		return nil, fmt.Errorf("Failed to execute the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	return data, nil
}

// WriteOutputFile function write data into a file.
func (g *OrbitGenerator) WriteOutputFile(outputPath string, data bytes.Buffer) error {
	file, err := os.Create(outputPath)
	if err != nil {
		return fmt.Errorf("Failed to create the output file %s:\n%s", outputPath, err)
	}

	_, err = file.Write(data.Bytes())
	if err != nil {
		return fmt.Errorf("Failed to write into the output file %s:\n%s", outputPath, err)
	}

	err = file.Close()
	if err != nil {
		return fmt.Errorf("Failed to close the writer:\n%s", err)
	}

	return nil
}
<commit_msg>typo in generator.go<commit_after>package generator

import (
	"bytes"
	"fmt"
	"html/template"
	"os"

	"github.com/gulien/orbit/context"
)

type (
	// OrbitGenerator struct helps to generate a file from a template.
	OrbitGenerator struct {
		// context is an instance of OrbitContext.
		context *context.OrbitContext
	}
)

// NewOrbitGenerator func instantiates a new instance of OrbitGenerator.
func NewOrbitGenerator(context *context.OrbitContext) *OrbitGenerator {
	return &OrbitGenerator{
		context,
	}
}

// Parse function parses a template and populates it.
func (g *OrbitGenerator) Parse() (bytes.Buffer, error) {
	tmpl := template.New(g.context.TemplateFilePath)
	tmpl, err := tmpl.ParseFiles(g.context.TemplateFilePath)

	if err != nil {
		return nil, fmt.Errorf("Failed to parse the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	var data bytes.Buffer
	if err := tmpl.Execute(&data, g.context); err != nil {
		return nil, fmt.Errorf("Failed to execute the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	return data, nil
}

// WriteOutputFile function writes data into a file.
func (g *OrbitGenerator) WriteOutputFile(outputPath string, data bytes.Buffer) error {
	file, err := os.Create(outputPath)
	if err != nil {
		return fmt.Errorf("Failed to create the output file %s:\n%s", outputPath, err)
	}

	_, err = file.Write(data.Bytes())
	if err != nil {
		return fmt.Errorf("Failed to write into the output file %s:\n%s", outputPath, err)
	}

	err = file.Close()
	if err != nil {
		return fmt.Errorf("Failed to close the writer:\n%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017, Project ArteMisc
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package generichash

import (
	"hash"

	"github.com/minio/blake2b-simd"
	"go.artemisc.eu/godium"
	"go.artemisc.eu/godium/internal"
)

const (
	Blake2b_BytesMin      = 16
	Blake2b_BytesMax      = 64
	Blake2b_Bytes         = 32
	Blake2b_KeyBytesMin   = 16
	Blake2b_KeyBytesMax   = 64
	Blake2b_KeyBytes      = 32
	Blake2b_SaltBytes     = 16
	Blake2b_PersonalBytes = 16

	size256 = 32
	size512 = 64
)

// Blake2b
type Blake2b struct {
	hash.Hash
}

// Blake2bSum256
func Blake2bSum256(data []byte) (sum [32]byte) {
	b := NewBlake2b256(nil)
	b.Write(data)
	b.Sum(sum[:0])
	return
}

// Blake2bSum512
func Blake2bSum512(data []byte) (sum [64]byte) {
	b := NewBlake2b512(nil)
	b.Write(data)
	b.Sum(sum[:0])
	return
}

// NewBlake2b256
func NewBlake2b256(key []byte) (gh godium.GenericHash) {
	var h hash.Hash
	if len(key) == 0 {
		h = blake2b.New256()
	} else {
		h = blake2b.NewMAC(size256, key)
	}
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2b512
func NewBlake2b512(key []byte) (gh godium.GenericHash) {
	var h hash.Hash
	if len(key) == 0 {
		h = blake2b.New512()
	} else {
		h = blake2b.NewMAC(size512, key)
	}
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2bXOF
func NewBlake2b(size uint32, key []byte) (gh godium.GenericHash) {
	h := blake2b.NewMAC(uint8(size), key)
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2bSaltPersonal
func NewBlake2bSaltPersonal(size uint32, key, personal, salt []byte) (b *Blake2b) {
	c := new(blake2b.Config)
	c.Size = uint8(size)
	c.Key = internal.Copy(key, uint64(len(key)))
	if personal != nil {

		c.Person = internal.Copy(personal, Blake2b_PersonalBytes)
	}
	if salt != nil {
		c.Salt = internal.Copy(salt, Blake2b_SaltBytes)
	}
	h, _ := blake2b.New(&blake2b.Config{})

	b = &Blake2b{
		Hash: h,
	}
	return
}

// Wipe
func (b *Blake2b) Wipe() {
	b.Hash.Reset()
}

func (b *Blake2b) BytesMin() int      { return Blake2b_BytesMin }
func (b *Blake2b) BytesMax() int      { return Blake2b_BytesMax }
func (b *Blake2b) Bytes() int         { return Blake2b_Bytes }
func (b *Blake2b) KeyBytesMin() int   { return Blake2b_KeyBytesMin }
func (b *Blake2b) KeyBytesMax() int   { return Blake2b_KeyBytesMax }
func (b *Blake2b) KeyBytes() int      { return Blake2b_KeyBytes }
func (b *Blake2b) PersonalBytes() int { return Blake2b_PersonalBytes }
func (b *Blake2b) SaltBytes() int     { return Blake2b_SaltBytes }
<commit_msg>Give the initialized blake2b.Config to blake2b.New<commit_after>// Copyright 2017, Project ArteMisc
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package generichash

import (
	"hash"

	"github.com/minio/blake2b-simd"
	"go.artemisc.eu/godium"
	"go.artemisc.eu/godium/internal"
)

const (
	Blake2b_BytesMin      = 16
	Blake2b_BytesMax      = 64
	Blake2b_Bytes         = 32
	Blake2b_KeyBytesMin   = 16
	Blake2b_KeyBytesMax   = 64
	Blake2b_KeyBytes      = 32
	Blake2b_SaltBytes     = 16
	Blake2b_PersonalBytes = 16

	size256 = 32
	size512 = 64
)

// Blake2b
type Blake2b struct {
	hash.Hash
}

// Blake2bSum256
func Blake2bSum256(data []byte) (sum [32]byte) {
	b := NewBlake2b256(nil)
	b.Write(data)
	b.Sum(sum[:0])
	return
}

// Blake2bSum512
func Blake2bSum512(data []byte) (sum [64]byte) {
	b := NewBlake2b512(nil)
	b.Write(data)
	b.Sum(sum[:0])
	return
}

// NewBlake2b256
func NewBlake2b256(key []byte) (gh godium.GenericHash) {
	var h hash.Hash
	if len(key) == 0 {
		h = blake2b.New256()
	} else {
		h = blake2b.NewMAC(size256, key)
	}
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2b512
func NewBlake2b512(key []byte) (gh godium.GenericHash) {
	var h hash.Hash
	if len(key) == 0 {
		h = blake2b.New512()
	} else {
		h = blake2b.NewMAC(size512, key)
	}
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2bXOF
func NewBlake2b(size uint32, key []byte) (gh godium.GenericHash) {
	h := blake2b.NewMAC(uint8(size), key)
	gh = &Blake2b{
		Hash: h,
	}
	return
}

// NewBlake2bSaltPersonal
func NewBlake2bSaltPersonal(size uint32, key, personal, salt []byte) (b *Blake2b) {
	c := new(blake2b.Config)
	c.Size = uint8(size)
	c.Key = internal.Copy(key, uint64(len(key)))
	if personal != nil {

		c.Person = internal.Copy(personal, Blake2b_PersonalBytes)
	}
	if salt != nil {
		c.Salt = internal.Copy(salt, Blake2b_SaltBytes)
	}
	h, _ := blake2b.New(c)

	b = &Blake2b{
		Hash: h,
	}
	return
}

// Wipe
func (b *Blake2b) Wipe() {
	b.Hash.Reset()
}

func (b *Blake2b) BytesMin() int      { return Blake2b_BytesMin }
func (b *Blake2b) BytesMax() int      { return Blake2b_BytesMax }
func (b *Blake2b) Bytes() int         { return Blake2b_Bytes }
func (b *Blake2b) KeyBytesMin() int   { return Blake2b_KeyBytesMin }
func (b *Blake2b) KeyBytesMax() int   { return Blake2b_KeyBytesMax }
func (b *Blake2b) KeyBytes() int      { return Blake2b_KeyBytes }
func (b *Blake2b) PersonalBytes() int { return Blake2b_PersonalBytes }
func (b *Blake2b) SaltBytes() int     { return Blake2b_SaltBytes }
<|endoftext|>


<commit_before>package topology

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"io"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.ServerAddress(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()

	waitTimeout := time.NewTimer(time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-waitTimeout.C:
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}

func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(true, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				stream, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId:    uint32(vid),
					Preallocate: preallocate,
				})
				if err != nil {
					return err
				}

				for {
					resp, recvErr := stream.Recv()
					if recvErr != nil {
						if recvErr == io.EOF {
							break
						} else {
							return recvErr
						}
					}
					glog.V(0).Infof("%d vacuum %d on %s processed %d bytes", index, vid, url, resp.ProcessedBytes)
				}
				return nil
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.ServerAddress(), vid)
	}
	isVacuumSuccess := true

	waitTimeout := time.NewTimer(3 * time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-waitTimeout.C:
			return false
		}
	}
	return isVacuumSuccess
}

func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	isReadOnly := false
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			if resp != nil && resp.IsReadOnly {
				isReadOnly = true
			}
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid, isReadOnly)
		}
	}
	return isCommitSuccess
}

func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, volumeId uint32, collection string, preallocate int64) {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f collection: %s volumeId: %d",
		garbageThreshold, collection, volumeId)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		if collection != "" && collection != c.Name {
			continue
		}
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				if volumeId > 0 && volumeLayout.Lookup(needle.VolumeId(volumeId)) == nil {
					continue
				}
				t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
			}
		}
	}
}

func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList.Copy()
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
		volumeLayout.accessLock.RUnlock()

		if isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
			if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				t.batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<commit_msg>fix filtering by volume id<commit_after>package topology

import (
	"context"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"io"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/storage/needle"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
)

func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
	ch := make(chan int, locationlist.Length())
	errCount := int32(0)
	for index, dn := range locationlist.list {
		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
			err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
					VolumeId: uint32(vid),
				})
				if err != nil {
					atomic.AddInt32(&errCount, 1)
					ch <- -1
					return err
				}
				if resp.GarbageRatio >= garbageThreshold {
					ch <- index
				} else {
					ch <- -1
				}
				return nil
			})
			if err != nil {
				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
			}
		}(index, dn.ServerAddress(), vid)
	}
	vacuumLocationList := NewVolumeLocationList()

	waitTimeout := time.NewTimer(time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case index := <-ch:
			if index != -1 {
				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
			}
		case <-waitTimeout.C:
			return vacuumLocationList, false
		}
	}
	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
}

func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
	locationlist *VolumeLocationList, preallocate int64) bool {
	vl.accessLock.Lock()
	vl.removeFromWritable(vid)
	vl.accessLock.Unlock()

	ch := make(chan bool, locationlist.Length())
	for index, dn := range locationlist.list {
		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
			err := operation.WithVolumeServerClient(true, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
				stream, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
					VolumeId:    uint32(vid),
					Preallocate: preallocate,
				})
				if err != nil {
					return err
				}

				for {
					resp, recvErr := stream.Recv()
					if recvErr != nil {
						if recvErr == io.EOF {
							break
						} else {
							return recvErr
						}
					}
					glog.V(0).Infof("%d vacuum %d on %s processed %d bytes", index, vid, url, resp.ProcessedBytes)
				}
				return nil
			})
			if err != nil {
				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
				ch <- false
			} else {
				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
				ch <- true
			}
		}(index, dn.ServerAddress(), vid)
	}
	isVacuumSuccess := true

	waitTimeout := time.NewTimer(3 * time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
	defer waitTimeout.Stop()

	for range locationlist.list {
		select {
		case canCommit := <-ch:
			isVacuumSuccess = isVacuumSuccess && canCommit
		case <-waitTimeout.C:
			return false
		}
	}
	return isVacuumSuccess
}

func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) bool {
	isCommitSuccess := true
	isReadOnly := false
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
				VolumeId: uint32(vid),
			})
			if resp != nil && resp.IsReadOnly {
				isReadOnly = true
			}
			return err
		})
		if err != nil {
			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
			isCommitSuccess = false
		} else {
			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
		}
	}
	if isCommitSuccess {
		for _, dn := range locationlist.list {
			vl.SetVolumeAvailable(dn, vid, isReadOnly)
		}
	}
	return isCommitSuccess
}

func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
	for _, dn := range locationlist.list {
		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
				VolumeId: uint32(vid),
			})
			return err
		})
		if err != nil {
			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
		} else {
			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
		}
	}
}

func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, volumeId uint32, collection string, preallocate int64) {

	// if there is vacuum going on, return immediately
	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
	if !swapped {
		return
	}
	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)

	// now only one vacuum process going on

	glog.V(1).Infof("Start vacuum on demand with threshold: %f collection: %s volumeId: %d",
		garbageThreshold, collection, volumeId)
	for _, col := range t.collectionMap.Items() {
		c := col.(*Collection)
		if collection != "" && collection != c.Name {
			continue
		}
		for _, vl := range c.storageType2VolumeLayout.Items() {
			if vl != nil {
				volumeLayout := vl.(*VolumeLayout)
				if volumeId > 0 {
					if volumeLayout.Lookup(needle.VolumeId(volumeId)) != nil {
						t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
					}
				} else {
					t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, preallocate)
				}
			}
		}
	}
}

func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, preallocate int64) {

	volumeLayout.accessLock.RLock()
	tmpMap := make(map[needle.VolumeId]*VolumeLocationList)
	for vid, locationList := range volumeLayout.vid2location {
		tmpMap[vid] = locationList.Copy()
	}
	volumeLayout.accessLock.RUnlock()

	for vid, locationList := range tmpMap {

		volumeLayout.accessLock.RLock()
		isReadOnly := volumeLayout.readonlyVolumes.IsTrue(vid)
		volumeLayout.accessLock.RUnlock()

		if isReadOnly {
			continue
		}

		glog.V(2).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
		if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
			if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
				t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			} else {
				t.batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
			}
		}
	}
}
<|endoftext|>


<commit_before>package local

import (
	"bytes"
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/plans"
	"github.com/hashicorp/terraform/plans/planfile"
	"github.com/hashicorp/terraform/states/statemgr"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/tfdiags"
)

func (b *Local) opPlan(
	stopCtx context.Context,
	cancelCtx context.Context,
	op *backend.Operation,
	runningOp *backend.RunningOperation) {

	log.Printf("[INFO] backend/local: starting Plan operation")

	var diags tfdiags.Diagnostics
	var err error

	if b.CLI != nil && op.PlanFile != nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Can't re-plan a saved plan",
			"The plan command was given a saved plan file as its input. This command generates a new plan, and so it requires a configuration directory as its argument.",
		))
		b.ReportResult(runningOp, diags)
		return
	}

	// Local planning requires a config, unless we're planning to destroy.
	if !op.Destroy && !op.HasConfig() {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"No configuration files",
			"Plan requires configuration to be present. Planning without a configuration would mark everything for destruction, which is normally not what is desired. If you would like to destroy everything, run plan with the -destroy option. Otherwise, create a Terraform configuration file (.tf file) and try again.",
		))
		b.ReportResult(runningOp, diags)
		return
	}

	// Setup our count hook that keeps track of resource changes
	countHook := new(CountHook)
	if b.ContextOpts == nil {
		b.ContextOpts = new(terraform.ContextOpts)
	}
	old := b.ContextOpts.Hooks
	defer func() { b.ContextOpts.Hooks = old }()
	b.ContextOpts.Hooks = append(b.ContextOpts.Hooks, countHook)

	// Get our context
	tfCtx, configSnap, opState, ctxDiags := b.context(op)
	diags = diags.Append(ctxDiags)
	if ctxDiags.HasErrors() {
		b.ReportResult(runningOp, diags)
		return
	}

	// Setup the state
	runningOp.State = tfCtx.State()

	// If we're refreshing before plan, perform that
	baseState := runningOp.State
	if op.PlanRefresh {
		log.Printf("[INFO] backend/local: plan calling Refresh")

		if b.CLI != nil {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planRefreshing) + "\n"))
		}

		refreshedState, err := tfCtx.Refresh()
		if err != nil {
			diags = diags.Append(err)
			b.ReportResult(runningOp, diags)
			return
		}
		baseState = refreshedState // plan will be relative to our refreshed state
		if b.CLI != nil {
			b.CLI.Output("\n------------------------------------------------------------------------")
		}
	}

	// Perform the plan in a goroutine so we can be interrupted
	var plan *plans.Plan
	var planDiags tfdiags.Diagnostics
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		log.Printf("[INFO] backend/local: plan calling Plan")
		plan, planDiags = tfCtx.Plan()
	}()

	if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
		return
	}

	diags = diags.Append(planDiags)
	if planDiags.HasErrors() {
		b.ReportResult(runningOp, diags)
		return
	}
	// Record state
	runningOp.PlanEmpty = plan.Changes.Empty()

	// Save the plan to disk
	if path := op.PlanOutPath; path != "" {
		plan.Backend = *op.PlanOutBackend

		// We may have updated the state in the refresh step above, but we
		// will freeze that updated state in the plan file for now and
		// only write it if this plan is subsequently applied.
		plannedStateFile := statemgr.PlannedStateUpdate(opState, baseState)

		log.Printf("[INFO] backend/local: writing plan output to: %s", path)
		err = planfile.Create(path, configSnap, plannedStateFile, plan)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Failed to write plan file",
				fmt.Sprintf("The plan file could not be written: %s.", err),
			))
			b.ReportResult(runningOp, diags)
			return
		}
	}

	// Perform some output tasks if we have a CLI to output to.
	if b.CLI != nil {
		schemas := tfCtx.Schemas()

		if plan.Changes.Empty() {
			b.CLI.Output("\n" + b.Colorize().Color(strings.TrimSpace(planNoChanges)))
			return
		}

		b.renderPlan(plan, schemas)

		// If we've accumulated any warnings along the way then we'll show them
		// here just before we show the summary and next steps. If we encountered
		// errors then we would've returned early at some other point above.
		b.ShowDiagnostics(diags)

		// Give the user some next-steps, unless we're running in an automation
		// tool which is presumed to provide its own UI for further actions.
		if !b.RunningInAutomation {

			b.CLI.Output("\n------------------------------------------------------------------------")

			if path := op.PlanOutPath; path == "" {
				b.CLI.Output(fmt.Sprintf(
					"\n" + strings.TrimSpace(planHeaderNoOutput) + "\n",
				))
			} else {
				b.CLI.Output(fmt.Sprintf(
					"\n"+strings.TrimSpace(planHeaderYesOutput)+"\n",
					path, path,
				))
			}

		}
	}
}

func (b *Local) renderPlan(plan *plans.Plan, schemas *terraform.Schemas) {

	counts := map[plans.Action]int{}
	for _, change := range plan.Changes.Resources {
		counts[change.Action]++
	}
	for _, change := range plan.Changes.RootOutputs {
		counts[change.Action]++
	}

	headerBuf := &bytes.Buffer{}
	fmt.Fprintf(headerBuf, "\n%s\n", strings.TrimSpace(planHeaderIntro))
	if counts[plans.Create] > 0 {
		fmt.Fprintf(headerBuf, "%s create\n", format.DiffActionSymbol(terraform.DiffCreate))
	}
	if counts[plans.Update] > 0 {
		fmt.Fprintf(headerBuf, "%s update in-place\n", format.DiffActionSymbol(terraform.DiffUpdate))
	}
	if counts[plans.Delete] > 0 {
		fmt.Fprintf(headerBuf, "%s destroy\n", format.DiffActionSymbol(terraform.DiffDestroy))
	}
	if counts[plans.Replace] > 0 {
		fmt.Fprintf(headerBuf, "%s destroy and then create replacement\n", format.DiffActionSymbol(terraform.DiffDestroyCreate))
	}
	if counts[plans.Read] > 0 {
		fmt.Fprintf(headerBuf, "%s read (data resources)\n", format.DiffActionSymbol(terraform.DiffRefresh))
	}

	b.CLI.Output(b.Colorize().Color(headerBuf.String()))

	b.CLI.Output("Terraform will perform the following actions:\n")

	for _, rcs := range plan.Changes.Resources {
		if rcs.Action == plans.NoOp {
			continue
		}
		providerSchema := schemas.ProviderSchema(rcs.ProviderAddr.ProviderConfig.Type)
		if providerSchema == nil {
			// Should never happen
			b.CLI.Output(fmt.Sprintf("(schema missing for %s)\n", rcs.ProviderAddr))
			continue
		}
		rSchema := providerSchema.SchemaForResourceAddr(rcs.Addr.Resource.Resource)
		if rSchema == nil {
			// Should never happen
			b.CLI.Output(fmt.Sprintf("(schema missing for %s)\n", rcs.Addr))
			continue
		}
		b.CLI.Output(format.ResourceChange(
			rcs,
			rSchema,
			b.CLIColor,
		))
	}

	// stats is similar to counts above, but:
	// - it considers only resource changes
	// - it simplifies "replace" into both a create and a delete
	stats := map[plans.Action]int{}
	for _, change := range plan.Changes.Resources {
		switch change.Action {
		case plans.Replace:
			counts[plans.Create]++
			counts[plans.Delete]++
		default:
			counts[change.Action]++
		}
	}
	b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
		"[reset][bold]Plan:[reset] "+
			"%d to add, %d to change, %d to destroy.",
		stats[plans.Create], stats[plans.Update], stats[plans.Delete],
	)))
}

const planErrNoConfig = `
No configuration files found!

Plan requires configuration to be present. Planning without a configuration
would mark everything for destruction, which is normally not what is desired.
If you would like to destroy everything, please run plan with the "-destroy"
flag or create a single empty configuration file. Otherwise, please create
a Terraform configuration file in the path being executed and try again.
`

const planHeaderIntro = `
An execution plan has been generated and is shown below.
Resource actions are indicated with the following symbols:
`

const planHeaderNoOutput = `
Note: You didn't specify an "-out" parameter to save this plan, so Terraform
can't guarantee that exactly these actions will be performed if
"terraform apply" is subsequently run.
`

const planHeaderYesOutput = `
This plan was saved to: %s

To perform exactly these actions, run the following command to apply:
    terraform apply %q
`

const planNoChanges = `
[reset][bold][green]No changes. Infrastructure is up-to-date.[reset][green]

This means that Terraform did not detect any differences between your
configuration and real physical resources that exist. As a result, no
actions need to be performed.
`

const planRefreshing = `
[reset][bold]Refreshing Terraform state in-memory prior to plan...[reset]
The refreshed state will be used to calculate this plan, but will not be
persisted to local or remote state storage.
`
<commit_msg>backend/local: Sort planned resource changes before rendering them<commit_after>package local

import (
	"bytes"
	"context"
	"fmt"
	"log"
	"sort"
	"strings"

	"github.com/hashicorp/terraform/backend"
	"github.com/hashicorp/terraform/command/format"
	"github.com/hashicorp/terraform/plans"
	"github.com/hashicorp/terraform/plans/planfile"
	"github.com/hashicorp/terraform/states/statemgr"
	"github.com/hashicorp/terraform/terraform"
	"github.com/hashicorp/terraform/tfdiags"
)

func (b *Local) opPlan(
	stopCtx context.Context,
	cancelCtx context.Context,
	op *backend.Operation,
	runningOp *backend.RunningOperation) {

	log.Printf("[INFO] backend/local: starting Plan operation")

	var diags tfdiags.Diagnostics
	var err error

	if b.CLI != nil && op.PlanFile != nil {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"Can't re-plan a saved plan",
			"The plan command was given a saved plan file as its input. This command generates a new plan, and so it requires a configuration directory as its argument.",
		))
		b.ReportResult(runningOp, diags)
		return
	}

	// Local planning requires a config, unless we're planning to destroy.
	if !op.Destroy && !op.HasConfig() {
		diags = diags.Append(tfdiags.Sourceless(
			tfdiags.Error,
			"No configuration files",
			"Plan requires configuration to be present. Planning without a configuration would mark everything for destruction, which is normally not what is desired. If you would like to destroy everything, run plan with the -destroy option. Otherwise, create a Terraform configuration file (.tf file) and try again.",
		))
		b.ReportResult(runningOp, diags)
		return
	}

	// Setup our count hook that keeps track of resource changes
	countHook := new(CountHook)
	if b.ContextOpts == nil {
		b.ContextOpts = new(terraform.ContextOpts)
	}
	old := b.ContextOpts.Hooks
	defer func() { b.ContextOpts.Hooks = old }()
	b.ContextOpts.Hooks = append(b.ContextOpts.Hooks, countHook)

	// Get our context
	tfCtx, configSnap, opState, ctxDiags := b.context(op)
	diags = diags.Append(ctxDiags)
	if ctxDiags.HasErrors() {
		b.ReportResult(runningOp, diags)
		return
	}

	// Setup the state
	runningOp.State = tfCtx.State()

	// If we're refreshing before plan, perform that
	baseState := runningOp.State
	if op.PlanRefresh {
		log.Printf("[INFO] backend/local: plan calling Refresh")

		if b.CLI != nil {
			b.CLI.Output(b.Colorize().Color(strings.TrimSpace(planRefreshing) + "\n"))
		}

		refreshedState, err := tfCtx.Refresh()
		if err != nil {
			diags = diags.Append(err)
			b.ReportResult(runningOp, diags)
			return
		}
		baseState = refreshedState // plan will be relative to our refreshed state
		if b.CLI != nil {
			b.CLI.Output("\n------------------------------------------------------------------------")
		}
	}

	// Perform the plan in a goroutine so we can be interrupted
	var plan *plans.Plan
	var planDiags tfdiags.Diagnostics
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)
		log.Printf("[INFO] backend/local: plan calling Plan")
		plan, planDiags = tfCtx.Plan()
	}()

	if b.opWait(doneCh, stopCtx, cancelCtx, tfCtx, opState) {
		return
	}

	diags = diags.Append(planDiags)
	if planDiags.HasErrors() {
		b.ReportResult(runningOp, diags)
		return
	}
	// Record state
	runningOp.PlanEmpty = plan.Changes.Empty()

	// Save the plan to disk
	if path := op.PlanOutPath; path != "" {
		plan.Backend = *op.PlanOutBackend

		// We may have updated the state in the refresh step above, but we
		// will freeze that updated state in the plan file for now and
		// only write it if this plan is subsequently applied.
		plannedStateFile := statemgr.PlannedStateUpdate(opState, baseState)

		log.Printf("[INFO] backend/local: writing plan output to: %s", path)
		err = planfile.Create(path, configSnap, plannedStateFile, plan)
		if err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Failed to write plan file",
				fmt.Sprintf("The plan file could not be written: %s.", err),
			))
			b.ReportResult(runningOp, diags)
			return
		}
	}

	// Perform some output tasks if we have a CLI to output to.
	if b.CLI != nil {
		schemas := tfCtx.Schemas()

		if plan.Changes.Empty() {
			b.CLI.Output("\n" + b.Colorize().Color(strings.TrimSpace(planNoChanges)))
			return
		}

		b.renderPlan(plan, schemas)

		// If we've accumulated any warnings along the way then we'll show them
		// here just before we show the summary and next steps. If we encountered
		// errors then we would've returned early at some other point above.
		b.ShowDiagnostics(diags)

		// Give the user some next-steps, unless we're running in an automation
		// tool which is presumed to provide its own UI for further actions.
		if !b.RunningInAutomation {

			b.CLI.Output("\n------------------------------------------------------------------------")

			if path := op.PlanOutPath; path == "" {
				b.CLI.Output(fmt.Sprintf(
					"\n" + strings.TrimSpace(planHeaderNoOutput) + "\n",
				))
			} else {
				b.CLI.Output(fmt.Sprintf(
					"\n"+strings.TrimSpace(planHeaderYesOutput)+"\n",
					path, path,
				))
			}

		}
	}
}

func (b *Local) renderPlan(plan *plans.Plan, schemas *terraform.Schemas) {

	counts := map[plans.Action]int{}
	for _, change := range plan.Changes.Resources {
		counts[change.Action]++
	}
	for _, change := range plan.Changes.RootOutputs {
		counts[change.Action]++
	}

	headerBuf := &bytes.Buffer{}
	fmt.Fprintf(headerBuf, "\n%s\n", strings.TrimSpace(planHeaderIntro))
	if counts[plans.Create] > 0 {
		fmt.Fprintf(headerBuf, "%s create\n", format.DiffActionSymbol(terraform.DiffCreate))
	}
	if counts[plans.Update] > 0 {
		fmt.Fprintf(headerBuf, "%s update in-place\n", format.DiffActionSymbol(terraform.DiffUpdate))
	}
	if counts[plans.Delete] > 0 {
		fmt.Fprintf(headerBuf, "%s destroy\n", format.DiffActionSymbol(terraform.DiffDestroy))
	}
	if counts[plans.Replace] > 0 {
		fmt.Fprintf(headerBuf, "%s destroy and then create replacement\n", format.DiffActionSymbol(terraform.DiffDestroyCreate))
	}
	if counts[plans.Read] > 0 {
		fmt.Fprintf(headerBuf, "%s read (data resources)\n", format.DiffActionSymbol(terraform.DiffRefresh))
	}

	b.CLI.Output(b.Colorize().Color(headerBuf.String()))

	b.CLI.Output("Terraform will perform the following actions:\n")

	// Note: we're modifying the backing slice of this plan object in-place
	// here. The ordering of resource changes in a plan is not significant,
	// but we can only do this safely here because we can assume that nobody
	// is concurrently modifying our changes while we're trying to print it.
	rChanges := plan.Changes.Resources
	sort.Slice(rChanges, func(i, j int) bool {
		iA := rChanges[i].Addr
		jA := rChanges[j].Addr
		if iA.String() == jA.String() {
			return rChanges[i].DeposedKey < rChanges[j].DeposedKey
		}
		return iA.Less(jA)
	})

	for _, rcs := range rChanges {
		if rcs.Action == plans.NoOp {
			continue
		}
		providerSchema := schemas.ProviderSchema(rcs.ProviderAddr.ProviderConfig.Type)
		if providerSchema == nil {
			// Should never happen
			b.CLI.Output(fmt.Sprintf("(schema missing for %s)\n", rcs.ProviderAddr))
			continue
		}
		rSchema := providerSchema.SchemaForResourceAddr(rcs.Addr.Resource.Resource)
		if rSchema == nil {
			// Should never happen
			b.CLI.Output(fmt.Sprintf("(schema missing for %s)\n", rcs.Addr))
			continue
		}
		b.CLI.Output(format.ResourceChange(
			rcs,
			rSchema,
			b.CLIColor,
		))
	}

	// stats is similar to counts above, but:
	// - it considers only resource changes
	// - it simplifies "replace" into both a create and a delete
	stats := map[plans.Action]int{}
	for _, change := range rChanges {
		switch change.Action {
		case plans.Replace:
			stats[plans.Create]++
			stats[plans.Delete]++
		default:
			stats[change.Action]++
		}
	}
	b.CLI.Output(b.Colorize().Color(fmt.Sprintf(
		"[reset][bold]Plan:[reset] "+
			"%d to add, %d to change, %d to destroy.",
		stats[plans.Create], stats[plans.Update], stats[plans.Delete],
	)))
}

const planErrNoConfig = `
No configuration files found!

Plan requires configuration to be present. Planning without a configuration
would mark everything for destruction, which is normally not what is desired.
If you would like to destroy everything, please run plan with the "-destroy"
flag or create a single empty configuration file. Otherwise, please create
a Terraform configuration file in the path being executed and try again.
`

const planHeaderIntro = `
An execution plan has been generated and is shown below.
Resource actions are indicated with the following symbols:
`

const planHeaderNoOutput = `
Note: You didn't specify an "-out" parameter to save this plan, so Terraform
can't guarantee that exactly these actions will be performed if
"terraform apply" is subsequently run.
`

const planHeaderYesOutput = `
This plan was saved to: %s

To perform exactly these actions, run the following command to apply:
    terraform apply %q
`

const planNoChanges = `
[reset][bold][green]No changes. Infrastructure is up-to-date.[reset][green]

This means that Terraform did not detect any differences between your
configuration and real physical resources that exist. As a result, no
actions need to be performed.
`

const planRefreshing = `
[reset][bold]Refreshing Terraform state in-memory prior to plan...[reset]
The refreshed state will be used to calculate this plan, but will not be
persisted to local or remote state storage.
`
<|endoftext|>