<commit_before>package main

import (
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"os/exec"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/fileSequence%d.ts", seq)
	return generated
}

func (pl PlaylistGenerator) GeneratedVideoFileForSequence(seq int) string {
	prefix := ""
	pref := client.Get("broadcast-prefix").Val()
	prefix = pref

	generated := fmt.Sprintf("fileSequence%d.ts", seq)
	postProcess := fmt.Sprintf("fileSequence%d-post.ts", seq)
	sourceVideo := prefix + generated
	destVideo := prefix + postProcess

	currentTime := time.Now().Format("3:04 PM")

	twoClipsAgo := seq - 2
	if twoClipsAgo > 0 {
		mapKey := fmt.Sprintf("/fileSequence%d-post.ts", twoClipsAgo)
		log.Println("map key is", mapKey)
		if count, ok := lfs.Counter[mapKey]; ok {
			currentTime = fmt.Sprintf("%d active viewers", count)
		}
	}

	err := RenderTextToPNG(currentTime, "time.png")
	if err == nil {
		cmd := exec.Command("avconv", "-i", sourceVideo, "-vf", "movie=time.png [watermark];[in][watermark] overlay=0:0 [out]", "-y", "-map", "0", "-c:a", "copy", "-c:v", "mpeg2video", "-an", destVideo)
		err := cmd.Start()
		if err != nil {
			return sourceVideo
		}
		err = cmd.Wait()
		return destVideo
	}

	return sourceVideo
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 728; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("fileSequence%d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 728; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<commit_msg>play pulp fiction<commit_after>package main

import (
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"os/exec"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/pulpfictionx/movie2m%5d.ts", seq)
	return generated
}

func (pl PlaylistGenerator) GeneratedVideoFileForSequence(seq int) string {
	prefix := ""
	pref := client.Get("broadcast-prefix").Val()
	prefix = pref

	generated := fmt.Sprintf("fileSequence%d.ts", seq)
	postProcess := fmt.Sprintf("fileSequence%d-post.ts", seq)
	sourceVideo := prefix + generated
	destVideo := prefix + postProcess

	currentTime := time.Now().Format("3:04 PM")

	twoClipsAgo := seq - 2
	if twoClipsAgo > 0 {
		mapKey := fmt.Sprintf("/fileSequence%d-post.ts", twoClipsAgo)
		log.Println("map key is", mapKey)
		if count, ok := lfs.Counter[mapKey]; ok {
			currentTime = fmt.Sprintf("%d active viewers", count)
		}
	}

	err := RenderTextToPNG(currentTime, "time.png")
	if err == nil {
		cmd := exec.Command("avconv", "-i", sourceVideo, "-vf", "movie=time.png [watermark];[in][watermark] overlay=0:0 [out]", "-y", "-map", "0", "-c:a", "copy", "-c:v", "mpeg2video", "-an", destVideo)
		err := cmd.Start()
		if err != nil {
			return sourceVideo
		}
		err = cmd.Wait()
		return destVideo
	}

	return sourceVideo
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 1854; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("movie2m%5d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 1854; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<|endoftext|>


<commit_before>package main

import (
	"broadcast/composite"
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"os/exec"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/media/truedetectives2e1movie%05d.ts", seq)
	return generated
}

func (pl PlaylistGenerator) GeneratedVideoFileForSequence(seq int) string {
	prefix := ""
	pref := client.Get("broadcast-prefix").Val()
	prefix = pref

	generated := fmt.Sprintf("fileSequence%d.ts", seq)
	postProcess := fmt.Sprintf("fileSequence%d-post.ts", seq)
	sourceVideo := prefix + generated
	destVideo := prefix + postProcess

	currentTime := time.Now().Format("3:04 PM")

	twoClipsAgo := seq - 2
	if twoClipsAgo > 0 {
		mapKey := fmt.Sprintf("/fileSequence%d-post.ts", twoClipsAgo)
		log.Println("map key is", mapKey)
		if count, ok := lfs.Counter[mapKey]; ok {
			currentTime = fmt.Sprintf("%d active viewers", count)
		}
	}

	err := composite.RenderTextToPNG(currentTime, "time.png", "font.ttf")
	if err == nil {
		cmd := exec.Command("avconv", "-i", sourceVideo, "-vf", "movie=time.png [watermark];[in][watermark] overlay=0:0 [out]", "-y", "-map", "0", "-c:a", "copy", "-c:v", "mpeg2video", "-an", destVideo)
		err := cmd.Start()
		if err != nil {
			return sourceVideo
		}
		err = cmd.Wait()
		return destVideo
	}

	return sourceVideo
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 1854; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("movie2m%5d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 1854; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<commit_msg>use full import path<commit_after>package main

import (
	"github.com/omarqazi/broadcast/composite"
	"fmt"
	"github.com/grafov/m3u8"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"os/exec"
	"time"
)

var broadcastCursor = make(chan int)
var currentPlaylist string
var client *redis.Client

func init() {
	client = redis.NewTCPClient(&redis.Options{
		Addr: "localhost:6379",
	})

	pong, err := client.Ping().Result()
	log.Println(pong, err)
}

type PlaylistGenerator struct {
	cursor chan int
}

func (pl PlaylistGenerator) VideoFileForSequence(seq int) string {
	generated := fmt.Sprintf("http://www.smick.tv/media/truedetectives2e1movie%05d.ts", seq)
	return generated
}

func (pl PlaylistGenerator) GeneratedVideoFileForSequence(seq int) string {
	prefix := ""
	pref := client.Get("broadcast-prefix").Val()
	prefix = pref

	generated := fmt.Sprintf("fileSequence%d.ts", seq)
	postProcess := fmt.Sprintf("fileSequence%d-post.ts", seq)
	sourceVideo := prefix + generated
	destVideo := prefix + postProcess

	currentTime := time.Now().Format("3:04 PM")

	twoClipsAgo := seq - 2
	if twoClipsAgo > 0 {
		mapKey := fmt.Sprintf("/fileSequence%d-post.ts", twoClipsAgo)
		log.Println("map key is", mapKey)
		if count, ok := lfs.Counter[mapKey]; ok {
			currentTime = fmt.Sprintf("%d active viewers", count)
		}
	}

	err := composite.RenderTextToPNG(currentTime, "time.png", "font.ttf")
	if err == nil {
		cmd := exec.Command("avconv", "-i", sourceVideo, "-vf", "movie=time.png [watermark];[in][watermark] overlay=0:0 [out]", "-y", "-map", "0", "-c:a", "copy", "-c:v", "mpeg2video", "-an", destVideo)
		err := cmd.Start()
		if err != nil {
			return sourceVideo
		}
		err = cmd.Wait()
		return destVideo
	}

	return sourceVideo
}

func (pl *PlaylistGenerator) KeepPlaylistUpdated() {
	p, e := m3u8.NewMediaPlaylist(1000, 1000)
	if e != nil {
		log.Println("Error creating media playlist:", e)
		return
	}
	currentPlaylist = p.Encode().String()

	for seqnum := 1; seqnum < 1854; seqnum = <-pl.cursor {
		videoFile := pl.VideoFileForSequence(seqnum)
		if err := p.Append(videoFile, 5.0, ""); err != nil {
			log.Println("Error appending item to playlist:", err, fmt.Sprintf("movie2m%5d.ts", seqnum))
		}
		currentPlaylist = p.Encode().String()
	}
}

func (pl *PlaylistGenerator) Start() {
	pl.cursor = make(chan int, 1000)

	go pl.KeepPlaylistUpdated()
	for i := 1; i < 1854; i++ {
		log.Println(i)
		pl.cursor <- i
		time.Sleep(5 * time.Second)
	}
}

func (pl PlaylistGenerator) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, currentPlaylist)
}
<|endoftext|>


<commit_before>// Application that captures webpage archives on a CT worker and uploads it to
// Google Storage.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/skia-dev/glog"

	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum     = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType   = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create from the Alexa CSV list. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Capturing Archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Delete and remake the local webpage archives directory.
	pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
	skutil.RemoveAll(pathToArchives)
	skutil.MkdirAll(pathToArchives, 0700)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build if it does not exist locally.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
	// Loop through all pagesets.
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}
	// TODO(rmistry): Remove this hack once the 1M webpage archives have been captured.
	glog.Infof("The length of fileInfos is: %s", len(fileInfos))
	fileInfos = fileInfos[6500:7500]
	glog.Infof("The fileInfos are: %s", fileInfos)
	for _, fileInfo := range fileInfos {
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the record_wpr binary.
		pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)
		args := []string{
			"--extra-browser-args=--disable-setuid-sandbox",
			"--browser=exact",
			"--browser-executable=" + chromiumBinary,
			fmt.Sprintf("%s_page_set", pagesetArchiveName),
			"--page-set-base-dir=" + pathToPagesets,
		}
		env := []string{
			fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
			"DISPLAY=:0",
		}
		skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
	}

	// Write timestamp to the webpage archives dir.
	skutil.LogErr(util.CreateTimestampFile(pathToArchives))

	// Upload webpage archives dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
}
<commit_msg>Capture CT archives from 7500-8499<commit_after>// Application that captures webpage archives on a CT worker and uploads it to
// Google Storage.
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"path/filepath"

	"github.com/skia-dev/glog"

	"strings"
	"time"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum     = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType   = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets to create from the Alexa CSV list. Eg: 10k, Mobile10k, All.")
	chromiumBuild = flag.String("chromium_build", "", "The chromium build to use for this capture_archives run.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Capturing Archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	skutil.LogErr(util.CreateTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES))
	defer util.DeleteTaskFile(util.ACTIVITY_CAPTURING_ARCHIVES)

	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}

	// Reset the local chromium checkout.
	if err := util.ResetCheckout(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not reset %s: %s", util.ChromiumSrcDir, err)
		return
	}
	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Delete and remake the local webpage archives directory.
	pathToArchives := filepath.Join(util.WebArchivesDir, *pagesetType)
	skutil.RemoveAll(pathToArchives)
	skutil.MkdirAll(pathToArchives, 0700)

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build if it does not exist locally.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)
	recordWprBinary := filepath.Join(util.TelemetryBinariesDir, util.BINARY_RECORD_WPR)
	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].CaptureArchivesTimeoutSecs
	// Loop through all pagesets.
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}
	// TODO(rmistry): Remove this hack once the 1M webpage archives have been captured.
	glog.Infof("The length of fileInfos is: %s", len(fileInfos))
	fileInfos = fileInfos[7500:8500]
	glog.Infof("The fileInfos are: %s", fileInfos)
	for _, fileInfo := range fileInfos {
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the record_wpr binary.
		pagesetArchiveName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)
		args := []string{
			"--extra-browser-args=--disable-setuid-sandbox",
			"--browser=exact",
			"--browser-executable=" + chromiumBinary,
			fmt.Sprintf("%s_page_set", pagesetArchiveName),
			"--page-set-base-dir=" + pathToPagesets,
		}
		env := []string{
			fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", pathToPagesets),
			"DISPLAY=:0",
		}
		skutil.LogErr(util.ExecuteCmd(recordWprBinary, args, env, time.Duration(timeoutSecs)*time.Second, nil, nil))
	}

	// Write timestamp to the webpage archives dir.
	skutil.LogErr(util.CreateTimestampFile(pathToArchives))

	// Upload webpage archives dir to Google Storage.
	if err := gs.UploadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"regexp"
	"os"
	"os/exec"
	"strconv"
	"text/template"
	"io/ioutil"
	"bytes"
)

func main() {
	// Parse the VM ID out of /proc/cmdline to see if we need to run
	re := regexp.MustCompile(`\bvmid=([0-9]+)\b`)

	cmdlineBytes, err := ioutil.ReadFile("/proc/cmdline")
	if err != nil {
		fmt.Println("Unable to read /proc/cmdline. Closing")
		os.Exit(1)
	}

	cmdline := string(cmdlineBytes)

	matches := re.FindStringSubmatch(cmdline)

	if (matches == nil || len(matches) != 2) {
		fmt.Println("We're not running on a provisioned VM. Nothing to do. Closing")
		os.Exit(0);
	}

	vmid, err := strconv.Atoi(matches[1])
	if err != nil {
		fmt.Println("Invalid vmid")
		os.Exit(1)
	}

	if vmid < 1 || vmid > 254 {
		fmt.Println("Invalid vmid, outside of 1-254 range. vmid given:", vmid)
	}

	fmt.Println("Running for vmid:" , vmid)
	initVM(vmid)
}

func initVM(vmid int) {
	// Configure networking
	const network = `config_eth0="10.0.{{.Id}}.25/24"
routes_eth0="default via 10.0.{{.Id}}.5"
dns_servers_eth0="10.0.{{.Id}}.5"
`

	type VMInformation struct {
		Id int
	}

	vmInfo := VMInformation{vmid}

	t := template.Must(template.New("net").Parse(network))

	var net bytes.Buffer
	t.Execute(&net, vmInfo)

	err := ioutil.WriteFile("/etc/conf.d/net", net.Bytes(), 0644)
	if err != nil {
		fmt.Println("Error writing /etc/conf.d/net")
		os.Exit(2)
	}

	// Restart network
	err = exec.Command("/etc/init.d/net.eth0", "restart").Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(3)
	}

	fmt.Println("Successfully initialized networking")
}
<commit_msg>Minor todo addition<commit_after>package main

import (
	"fmt"
	"regexp"
	"os"
	"os/exec"
	"strconv"
	"text/template"
	"io/ioutil"
	"bytes"
)

func main() {
	// Parse the VM ID out of /proc/cmdline to see if we need to run
	re := regexp.MustCompile(`\bvmid=([0-9]+)\b`)

	cmdlineBytes, err := ioutil.ReadFile("/proc/cmdline")
	if err != nil {
		fmt.Println("Unable to read /proc/cmdline. Closing")
		os.Exit(1)
	}

	cmdline := string(cmdlineBytes)

	matches := re.FindStringSubmatch(cmdline)

	if (matches == nil || len(matches) != 2) {
		fmt.Println("We're not running on a provisioned VM. Nothing to do. Closing")
		os.Exit(0);
	}

	vmid, err := strconv.Atoi(matches[1])
	if err != nil {
		fmt.Println("Invalid vmid")
		os.Exit(1)
	}

	if vmid < 1 || vmid > 254 {
		fmt.Println("Invalid vmid, outside of 1-254 range. vmid given:", vmid)
	}

	fmt.Println("Running for vmid:" , vmid)
	initVM(vmid)
}

func initVM(vmid int) {
	// Configure networking
	const network = `config_eth0="10.0.{{.Id}}.25/24"
routes_eth0="default via 10.0.{{.Id}}.5"
dns_servers_eth0="10.0.{{.Id}}.5"
`

	type VMInformation struct {
		Id int
	}

	vmInfo := VMInformation{vmid}

	t := template.Must(template.New("net").Parse(network))

	var net bytes.Buffer
	// TODO Handle the case where there's an error parsing the template
	t.Execute(&net, vmInfo)

	err := ioutil.WriteFile("/etc/conf.d/net", net.Bytes(), 0644)
	if err != nil {
		fmt.Println("Error writing /etc/conf.d/net")
		os.Exit(2)
	}

	// Restart network
	err = exec.Command("/etc/init.d/net.eth0", "restart").Run()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(3)
	}

	fmt.Println("Successfully initialized networking")
}
<|endoftext|>


<commit_before>package falconPortal

import (
	"fmt"

	"strings"

	"github.com/Cepave/fe/g"
	"github.com/Cepave/fe/model/uic"
	"github.com/astaxie/beego/orm"
)

//generate status filter SQL templete
func genStatusQueryTemplete(status string, feildsName string, flag bool) (filterTemplete string, vflag bool) {
	if flag {
		filterTemplete = fmt.Sprintf(" AND %v ", filterTemplete)
	}
	vflag = true
	filterTemplete = fmt.Sprintf("%v (", filterTemplete)
	status_list := strings.Split(status, ",")
	for idx, s := range status_list {
		if idx == len(status_list)-1 {
			filterTemplete = fmt.Sprintf("%v %s = '%s')", filterTemplete, feildsName, s)
		} else {
			filterTemplete = fmt.Sprintf("%v %s = '%s' or ", filterTemplete, feildsName, s)
		}
	}
	return filterTemplete, vflag
}

func GetEventCases(startTime int64, endTime int64, priority int, status string, progressStatus string, limit int, elimit int, username string, metrics string) (result []EventCases, err error) {
	config := g.Config()
	if limit == 0 || limit > config.FalconPortal.Limit {
		limit = config.FalconPortal.Limit
	}

	isadmin, tplids, err := GetCasePermission(username)
	if tplids == "" {
		tplids = "-1"
	}
	q := orm.NewOrm()
	q.Using("falcon_portal")
	flag := false
	queryTmp := ""
	if startTime != 0 && endTime != 0 {
		flag = true
		queryTmp = fmt.Sprintf(" %v update_at >= %d AND  update_at <= %d", queryTmp, startTime, endTime)
	}
	if priority != -1 {
		if flag {
			queryTmp = fmt.Sprintf("%v and priority = %d", queryTmp, priority)
		} else {
			flag = true
			queryTmp = fmt.Sprintf("%v priority = %d", queryTmp, priority)
		}
	}
	if status != "ALL" {
		var statusFilterTmp string
		statusFilterTmp, flag = genStatusQueryTemplete(status, "status", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, statusFilterTmp)
	}
	if progressStatus != "ALL" {
		var progressFilterTmp string
		progressFilterTmp, flag = genStatusQueryTemplete(progressStatus, "process_status", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, progressFilterTmp)
	}

	if metrics != "ALL" {
		var metricFilterTmp string
		metricFilterTmp, flag = genStatusQueryTemplete(metrics, "metric", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, metricFilterTmp)
	}

	if queryTmp != "" && !isadmin {
		_, err = q.Raw(fmt.Sprintf("SELECT * FROM `event_cases` WHERE (tpl_creator = '%s' OR template_id in (%s)) AND %v order by update_at DESC limit %d", username, tplids, queryTmp, limit)).QueryRows(&result)
	} else {
		if queryTmp != "" {
			queryTmp = fmt.Sprintf("WHERE %v", queryTmp)
		}
		_, err = q.Raw(fmt.Sprintf("SELECT * FROM `event_cases` %v order by update_at DESC limit %d", queryTmp, limit)).QueryRows(&result)
	}

	if len(result) == 0 {
		result = []EventCases{}
	} else {
		var eventLimit int
		if eventLimit = elimit; elimit == 0 {
			eventLimit = 10
		}
		for indx, event := range result {
			var eventArr []*Events
			q.Raw(fmt.Sprintf("SELECT * FROM `events` WHERE event_caseId = '%s' order by timestamp DESC Limit %d", event.Id, eventLimit)).QueryRows(&eventArr)
			if len(eventArr) != 0 {
				event.Events = eventArr
			} else {
				event.Events = []*Events{}
			}
			result[indx] = event
		}
	}
	return
}

func GetEvents(startTime int64, endTime int64, status string, limit int) (result []EventsRsp, err error) {
	config := g.Config()
	if limit == 0 || limit > config.FalconPortal.Limit {
		limit = config.FalconPortal.Limit
	}

	q := orm.NewOrm()
	q.Using("falcon_portal")
	queryTmp := ""
	if status != "ALL" {
		queryTmp = fmt.Sprintf(" %v status = '%s' and ", queryTmp, status)
	}
	if startTime != 0 && endTime != 0 {
		queryTmp = fmt.Sprintf(" %v events.timestamp >= %d and events.timestamp <= %d", queryTmp, startTime, endTime)
	}
	if queryTmp != "" {
		_, err = q.Raw(fmt.Sprintf(`SELECT events.id as id,
					events.step as step,
					events.cond as cond,
					events.timestamp as timestamp,
					events.event_caseId as eid,
					event_cases.tpl_creator as tpl_creator,
					event_cases.metric as metric,
					event_cases.endpoint as endpoint
					FROM events LEFT JOIN event_cases on event_cases.id = events.event_caseId
					WHERE %v ORDER BY events.timestamp DESC limit %d`, queryTmp, limit)).QueryRows(&result)
	} else {
		_, err = q.Raw(fmt.Sprintf(`SELECT
					events.id as id,
					events.step as step,
					events.cond as cond,
					events.timestamp as timestamp,
					events.event_caseId as eid,
					event_cases.tpl_creator as tpl_creator,
					event_cases.metric as metric,
					event_cases.endpoint as endpoint
					FROM events LEFT JOIN event_cases on event_cases.id = events.event_caseId
					ORDER BY events.timestamp DESC limit %d`, limit)).QueryRows(&result)
	}
	if len(result) == 0 {
		result = []EventsRsp{}
	}
	return
}

func CountNumOfTlp() (c int, err error) {
	var h []Tpl
	q := orm.NewOrm()
	q.Using("falcon_portal")
	_, err = q.Raw("select * from `tpl`").QueryRows(&h)
	c = len(h)
	return
}

func GetNotes(event_caseId string, limit int) (enotes []EventNote, err error) {
	if limit == 0 {
		limit = 10
	}
	q := orm.NewOrm()
	q.Using("falcon_portal")
	_, err = q.Raw(fmt.Sprintf(`SELECT event_note.id as id,
				event_note.event_caseId as event_caseId,
				event_note.note as note,
				event_note.case_id as case_id,
				event_note.event_caseId as eid,
				event_note.status as status,
				event_note.timestamp as timestamp,
				user.name as user_name
				FROM falcon_portal.event_note as event_note LEFT JOIN uic.user as user on event_note.user_id = user.id
				WHERE event_note.event_caseId = '%s' ORDER BY event_note.timestamp DESC limit %d`, event_caseId, limit)).QueryRows(&enotes)
	return
}

func GetNote(noteId int64) (EventNote, error) {
	q := orm.NewOrm()
	q.Using("falcon_portal")
	var eventNote EventNote
	err := q.Raw(`SELECT * from event_note WHERE event_note.id = ?`, noteId).QueryRow(&eventNote)
	if err == nil {
		user := uic.ReadUserById(eventNote.UserId)
		eventNote.UserName = user.Name
	}
	return eventNote, err
}
<commit_msg>[OWL-658] fixed null event_notes reponse issue<commit_after>package falconPortal

import (
	"fmt"

	"strings"

	"github.com/Cepave/fe/g"
	"github.com/Cepave/fe/model/uic"
	"github.com/astaxie/beego/orm"
)

//generate status filter SQL templete
func genStatusQueryTemplete(status string, feildsName string, flag bool) (filterTemplete string, vflag bool) {
	if flag {
		filterTemplete = fmt.Sprintf(" AND %v ", filterTemplete)
	}
	vflag = true
	filterTemplete = fmt.Sprintf("%v (", filterTemplete)
	status_list := strings.Split(status, ",")
	for idx, s := range status_list {
		if idx == len(status_list)-1 {
			filterTemplete = fmt.Sprintf("%v %s = '%s')", filterTemplete, feildsName, s)
		} else {
			filterTemplete = fmt.Sprintf("%v %s = '%s' or ", filterTemplete, feildsName, s)
		}
	}
	return filterTemplete, vflag
}

func GetEventCases(startTime int64, endTime int64, priority int, status string, progressStatus string, limit int, elimit int, username string, metrics string) (result []EventCases, err error) {
	config := g.Config()
	if limit == 0 || limit > config.FalconPortal.Limit {
		limit = config.FalconPortal.Limit
	}

	isadmin, tplids, err := GetCasePermission(username)
	if tplids == "" {
		tplids = "-1"
	}
	q := orm.NewOrm()
	q.Using("falcon_portal")
	flag := false
	queryTmp := ""
	if startTime != 0 && endTime != 0 {
		flag = true
		queryTmp = fmt.Sprintf(" %v update_at >= %d AND  update_at <= %d", queryTmp, startTime, endTime)
	}
	if priority != -1 {
		if flag {
			queryTmp = fmt.Sprintf("%v and priority = %d", queryTmp, priority)
		} else {
			flag = true
			queryTmp = fmt.Sprintf("%v priority = %d", queryTmp, priority)
		}
	}
	if status != "ALL" {
		var statusFilterTmp string
		statusFilterTmp, flag = genStatusQueryTemplete(status, "status", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, statusFilterTmp)
	}
	if progressStatus != "ALL" {
		var progressFilterTmp string
		progressFilterTmp, flag = genStatusQueryTemplete(progressStatus, "process_status", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, progressFilterTmp)
	}

	if metrics != "ALL" {
		var metricFilterTmp string
		metricFilterTmp, flag = genStatusQueryTemplete(metrics, "metric", flag)
		queryTmp = fmt.Sprintf("%v %s", queryTmp, metricFilterTmp)
	}

	if queryTmp != "" && !isadmin {
		_, err = q.Raw(fmt.Sprintf("SELECT * FROM `event_cases` WHERE (tpl_creator = '%s' OR template_id in (%s)) AND %v order by update_at DESC limit %d", username, tplids, queryTmp, limit)).QueryRows(&result)
	} else {
		if queryTmp != "" {
			queryTmp = fmt.Sprintf("WHERE %v", queryTmp)
		}
		_, err = q.Raw(fmt.Sprintf("SELECT * FROM `event_cases` %v order by update_at DESC limit %d", queryTmp, limit)).QueryRows(&result)
	}

	if len(result) == 0 {
		result = []EventCases{}
	} else {
		var eventLimit int
		if eventLimit = elimit; elimit == 0 {
			eventLimit = 10
		}
		for indx, event := range result {
			var eventArr []*Events
			q.Raw(fmt.Sprintf("SELECT * FROM `events` WHERE event_caseId = '%s' order by timestamp DESC Limit %d", event.Id, eventLimit)).QueryRows(&eventArr)
			if len(eventArr) != 0 {
				event.Events = eventArr
			} else {
				event.Events = []*Events{}
			}
			result[indx] = event
		}
	}
	return
}

func GetEvents(startTime int64, endTime int64, status string, limit int) (result []EventsRsp, err error) {
	config := g.Config()
	if limit == 0 || limit > config.FalconPortal.Limit {
		limit = config.FalconPortal.Limit
	}

	q := orm.NewOrm()
	q.Using("falcon_portal")
	queryTmp := ""
	if status != "ALL" {
		queryTmp = fmt.Sprintf(" %v status = '%s' and ", queryTmp, status)
	}
	if startTime != 0 && endTime != 0 {
		queryTmp = fmt.Sprintf(" %v events.timestamp >= %d and events.timestamp <= %d", queryTmp, startTime, endTime)
	}
	if queryTmp != "" {
		_, err = q.Raw(fmt.Sprintf(`SELECT events.id as id,
					events.step as step,
					events.cond as cond,
					events.timestamp as timestamp,
					events.event_caseId as eid,
					event_cases.tpl_creator as tpl_creator,
					event_cases.metric as metric,
					event_cases.endpoint as endpoint
					FROM events LEFT JOIN event_cases on event_cases.id = events.event_caseId
					WHERE %v ORDER BY events.timestamp DESC limit %d`, queryTmp, limit)).QueryRows(&result)
	} else {
		_, err = q.Raw(fmt.Sprintf(`SELECT
					events.id as id,
					events.step as step,
					events.cond as cond,
					events.timestamp as timestamp,
					events.event_caseId as eid,
					event_cases.tpl_creator as tpl_creator,
					event_cases.metric as metric,
					event_cases.endpoint as endpoint
					FROM events LEFT JOIN event_cases on event_cases.id = events.event_caseId
					ORDER BY events.timestamp DESC limit %d`, limit)).QueryRows(&result)
	}
	if len(result) == 0 {
		result = []EventsRsp{}
	}
	return
}

func CountNumOfTlp() (c int, err error) {
	var h []Tpl
	q := orm.NewOrm()
	q.Using("falcon_portal")
	_, err = q.Raw("select * from `tpl`").QueryRows(&h)
	c = len(h)
	return
}

func GetNotes(event_caseId string, limit int) (enotes []EventNote, err error) {
	if limit == 0 {
		limit = 10
	}
	q := orm.NewOrm()
	q.Using("falcon_portal")
	_, err = q.Raw(fmt.Sprintf(`SELECT event_note.id as id,
				event_note.event_caseId as event_caseId,
				event_note.note as note,
				event_note.case_id as case_id,
				event_note.event_caseId as eid,
				event_note.status as status,
				event_note.timestamp as timestamp,
				user.name as user_name
				FROM falcon_portal.event_note as event_note LEFT JOIN uic.user as user on event_note.user_id = user.id
				WHERE event_note.event_caseId = '%s' ORDER BY event_note.timestamp DESC limit %d`, event_caseId, limit)).QueryRows(&enotes)
	if len(enotes) == 0 {
		enotes = []EventNote{}
	}
	return
}

func GetNote(noteId int64) (EventNote, error) {
	q := orm.NewOrm()
	q.Using("falcon_portal")
	var eventNote EventNote
	err := q.Raw(`SELECT * from event_note WHERE event_note.id = ?`, noteId).QueryRow(&eventNote)
	if err == nil {
		user := uic.ReadUserById(eventNote.UserId)
		eventNote.UserName = user.Name
	}
	return eventNote, err
}
<|endoftext|>


<commit_before>package main

const dockerProfileTemplate = `@{DOCKER_GRAPH_PATH}=/var/lib/docker

profile /usr/bin/docker (attach_disconnected, complain) {
  # Prevent following links to these files during container setup.
  deny /etc/** mkl,
  deny /dev/** kl,
  deny /sys/** mkl,
  deny /proc/** mkl,

  mount -> @{DOCKER_GRAPH_PATH}/**,
  mount -> /,
  mount -> /proc/**,
  mount -> /sys/**,
  mount -> /run/docker/netns/**,
  mount -> /.pivot_root[0-9]*/,

  / r,

  umount,
  pivot_root,
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  signal (receive) peer=@{profile_name},
  signal (receive) peer=unconfined,
  signal (send),
{{end}}{{end}}
  ipc rw,
  network,
  capability,
  owner /** rw,
  @{DOCKER_GRAPH_PATH}/** rwl,
  @{DOCKER_GRAPH_PATH}/linkgraph.db k,
  @{DOCKER_GRAPH_PATH}/network/files/boltdb.db k,
  @{DOCKER_GRAPH_PATH}/network/files/local-kv.db k,

  # For non-root client use:
  /dev/urandom r,
  /run/docker.sock rw,
  /proc/** r,
  /sys/kernel/mm/hugepages/ r,
  /etc/localtime r,
  /etc/ld.so.cache r,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  ptrace peer=@{profile_name},
  ptrace (read) peer=docker-default,
  deny ptrace (trace) peer=docker-default,
  deny ptrace peer=/usr/bin/docker///bin/ps,
{{end}}{{end}}

  /usr/lib/** rm,
  /lib/** rm,

  /usr/bin/docker pix,
  /sbin/xtables-multi rCx,
  /sbin/iptables rCx,
  /sbin/modprobe rCx,
  /sbin/auplink rCx,
  /sbin/mke2fs rCx,
  /sbin/tune2fs rCx,
  /sbin/blkid rCx,
  /bin/kmod rCx,
  /usr/bin/xz rCx,
  /bin/ps rCx,
  /bin/cat rCx,
  /sbin/zfs rCx,
  /sbin/apparmor_parser rCx,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  # Transitions
  change_profile -> docker-*,
  change_profile -> unconfined,
{{end}}{{end}}

  profile /bin/cat (complain) {
    /etc/ld.so.cache r,
    /lib/** rm,
    /dev/null rw,
    /proc r,
    /bin/cat mr,

    # For reading in 'docker stats':
    /proc/[0-9]*/net/dev r,
  }
  profile /bin/ps (complain) {
    /etc/ld.so.cache r,
    /etc/localtime r,
    /etc/passwd r,
    /etc/nsswitch.conf r,
    /lib/** rm,
    /proc/[0-9]*/** r,
    /dev/null rw,
    /bin/ps mr,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    # We don't need ptrace so we'll deny and ignore the error.
    deny ptrace (read, trace),
{{end}}{{end}}

    # Quiet dac_override denials
    deny capability dac_override,
    deny capability dac_read_search,
    deny capability sys_ptrace,

    /dev/tty r,
    /proc/stat r,
    /proc/cpuinfo r,
    /proc/meminfo r,
    /proc/uptime r,
    /sys/devices/system/cpu/online r,
    /proc/sys/kernel/pid_max r,
    /proc/ r,
    /proc/tty/drivers r,
  }
  profile /sbin/iptables (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability net_admin,
  }
  profile /sbin/auplink flags=(attach_disconnected, complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability sys_admin,
    capability dac_override,

    @{DOCKER_GRAPH_PATH}/aufs/** rw,
    @{DOCKER_GRAPH_PATH}/tmp/** rw,
    # For user namespaces:
    @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/** rw,

    /sys/fs/aufs/** r,
    /lib/** rm,
    /apparmor/.null r,
    /dev/null rw,
    /etc/ld.so.cache r,
    /sbin/auplink rm,
    /proc/fs/aufs/** rw,
    /proc/[0-9]*/mounts rw,
  }
  profile /sbin/modprobe /bin/kmod (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability sys_module,
    /etc/ld.so.cache r,
    /lib/** rm,
    /dev/null rw,
    /apparmor/.null rw,
    /sbin/modprobe rm,
    /bin/kmod rm,
    /proc/cmdline r,
    /sys/module/** r,
    /etc/modprobe.d{/,/**} r,
  }
  # xz works via pipes, so we do not need access to the filesystem.
  profile /usr/bin/xz (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    /etc/ld.so.cache r,
    /lib/** rm,
    /usr/bin/xz rm,
    deny /proc/** rw,
    deny /sys/** rw,
  }
  profile /sbin/xtables-multi (attach_disconnected, complain) {
    /etc/ld.so.cache r,
    /lib/** rm,
    /sbin/xtables-multi rm,
    /apparmor/.null w,
    /dev/null rw,

    /proc r,

    capability net_raw,
    capability net_admin,
    network raw,
  }
  profile /sbin/zfs (attach_disconnected, complain) {
    file,
    capability,
  }
  profile /sbin/mke2fs (complain) {
    /sbin/mke2fs rm,

    /lib/** rm,

    /apparmor/.null w,

    /etc/ld.so.cache r,
    /etc/mke2fs.conf r,
    /etc/mtab r,

    /dev/dm-* rw,
    /dev/urandom r,
    /dev/null rw,

    /proc/swaps r,
    /proc/[0-9]*/mounts r,
  }
  profile /sbin/tune2fs (complain) {
    /sbin/tune2fs rm,

    /lib/** rm,

    /apparmor/.null w,

    /etc/blkid.conf r,
    /etc/mtab r,
    /etc/ld.so.cache r,

    /dev/null rw,
    /dev/.blkid.tab r,
    /dev/dm-* rw,

    /proc/swaps r,
    /proc/[0-9]*/mounts r,
  }
  profile /sbin/blkid (complain) {
    /sbin/blkid rm,

    /lib/** rm,
    /apparmor/.null w,

    /etc/ld.so.cache r,
    /etc/blkid.conf r,

    /dev/null rw,
    /dev/.blkid.tab rl,
    /dev/.blkid.tab* rwl,
    /dev/dm-* r,

    /sys/devices/virtual/block/** r,

    capability mknod,

    mount -> @{DOCKER_GRAPH_PATH}/**,
  }
  profile /sbin/apparmor_parser (complain) {
    /sbin/apparmor_parser rm,

    /lib/** rm,

    /etc/ld.so.cache r,
    /etc/apparmor/** r,
    /etc/apparmor.d/** r,
    /etc/apparmor.d/cache/** w,

    /dev/null rw,

    /sys/kernel/security/apparmor/** r,
    /sys/kernel/security/apparmor/.replace w,

    /proc/[0-9]*/mounts r,
    /proc/sys/kernel/osrelease r,
    /proc r,

    capability mac_admin,
  }
}`
<commit_msg>Policy extensions for user namespaces and docker exec<commit_after>package main

const dockerProfileTemplate = `@{DOCKER_GRAPH_PATH}=/var/lib/docker

profile /usr/bin/docker (attach_disconnected, complain) {
  # Prevent following links to these files during container setup.
  deny /etc/** mkl,
  deny /dev/** kl,
  deny /sys/** mkl,
  deny /proc/** mkl,

  mount -> @{DOCKER_GRAPH_PATH}/**,
  mount -> /,
  mount -> /proc/**,
  mount -> /sys/**,
  mount -> /run/docker/netns/**,
  mount -> /.pivot_root[0-9]*/,

  / r,

  umount,
  pivot_root,
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  signal (receive) peer=@{profile_name},
  signal (receive) peer=unconfined,
  signal (send),
{{end}}{{end}}
  ipc rw,
  network,
  capability,
  owner /** rw,
  @{DOCKER_GRAPH_PATH}/** rwl,
  @{DOCKER_GRAPH_PATH}/linkgraph.db k,
  @{DOCKER_GRAPH_PATH}/network/files/boltdb.db k,
  @{DOCKER_GRAPH_PATH}/network/files/local-kv.db k,
  @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/linkgraph.db k,

  # For non-root client use:
  /dev/urandom r,
  /dev/null rw,
  /dev/pts/[0-9]* rw,
  /run/docker.sock rw,
  /proc/** r,
  /proc/[0-9]*/attr/exec w,
  /sys/kernel/mm/hugepages/ r,
  /etc/localtime r,
  /etc/ld.so.cache r,
  /etc/passwd r,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  ptrace peer=@{profile_name},
  ptrace (read) peer=docker-default,
  deny ptrace (trace) peer=docker-default,
  deny ptrace peer=/usr/bin/docker///bin/ps,
{{end}}{{end}}

  /usr/lib/** rm,
  /lib/** rm,

  /usr/bin/docker pix,
  /sbin/xtables-multi rCx,
  /sbin/iptables rCx,
  /sbin/modprobe rCx,
  /sbin/auplink rCx,
  /sbin/mke2fs rCx,
  /sbin/tune2fs rCx,
  /sbin/blkid rCx,
  /bin/kmod rCx,
  /usr/bin/xz rCx,
  /bin/ps rCx,
  /bin/cat rCx,
  /sbin/zfs rCx,
  /sbin/apparmor_parser rCx,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
  # Transitions
  change_profile -> docker-*,
  change_profile -> unconfined,
{{end}}{{end}}

  profile /bin/cat (complain) {
    /etc/ld.so.cache r,
    /lib/** rm,
    /dev/null rw,
    /proc r,
    /bin/cat mr,

    # For reading in 'docker stats':
    /proc/[0-9]*/net/dev r,
  }
  profile /bin/ps (complain) {
    /etc/ld.so.cache r,
    /etc/localtime r,
    /etc/passwd r,
    /etc/nsswitch.conf r,
    /lib/** rm,
    /proc/[0-9]*/** r,
    /dev/null rw,
    /bin/ps mr,

{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    # We don't need ptrace so we'll deny and ignore the error.
    deny ptrace (read, trace),
{{end}}{{end}}

    # Quiet dac_override denials
    deny capability dac_override,
    deny capability dac_read_search,
    deny capability sys_ptrace,

    /dev/tty r,
    /proc/stat r,
    /proc/cpuinfo r,
    /proc/meminfo r,
    /proc/uptime r,
    /sys/devices/system/cpu/online r,
    /proc/sys/kernel/pid_max r,
    /proc/ r,
    /proc/tty/drivers r,
  }
  profile /sbin/iptables (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability net_admin,
  }
  profile /sbin/auplink flags=(attach_disconnected, complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability sys_admin,
    capability dac_override,

    @{DOCKER_GRAPH_PATH}/aufs/** rw,
    @{DOCKER_GRAPH_PATH}/tmp/** rw,
    # For user namespaces:
    @{DOCKER_GRAPH_PATH}/[0-9]*.[0-9]*/** rw,

    /sys/fs/aufs/** r,
    /lib/** rm,
    /apparmor/.null r,
    /dev/null rw,
    /etc/ld.so.cache r,
    /sbin/auplink rm,
    /proc/fs/aufs/** rw,
    /proc/[0-9]*/mounts rw,
  }
  profile /sbin/modprobe /bin/kmod (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    capability sys_module,
    /etc/ld.so.cache r,
    /lib/** rm,
    /dev/null rw,
    /apparmor/.null rw,
    /sbin/modprobe rm,
    /bin/kmod rm,
    /proc/cmdline r,
    /sys/module/** r,
    /etc/modprobe.d{/,/**} r,
  }
  # xz works via pipes, so we do not need access to the filesystem.
  profile /usr/bin/xz (complain) {
{{if ge .MajorVersion 2}}{{if ge .MinorVersion 9}}
    signal (receive) peer=/usr/bin/docker,
{{end}}{{end}}
    /etc/ld.so.cache r,
    /lib/** rm,
    /usr/bin/xz rm,
    deny /proc/** rw,
    deny /sys/** rw,
  }
  profile /sbin/xtables-multi (attach_disconnected, complain) {
    /etc/ld.so.cache r,
    /lib/** rm,
    /sbin/xtables-multi rm,
    /apparmor/.null w,
    /dev/null rw,

    /proc r,

    capability net_raw,
    capability net_admin,
    network raw,
  }
  profile /sbin/zfs (attach_disconnected, complain) {
    file,
    capability,
  }
  profile /sbin/mke2fs (complain) {
    /sbin/mke2fs rm,

    /lib/** rm,

    /apparmor/.null w,

    /etc/ld.so.cache r,
    /etc/mke2fs.conf r,
    /etc/mtab r,

    /dev/dm-* rw,
    /dev/urandom r,
    /dev/null rw,

    /proc/swaps r,
    /proc/[0-9]*/mounts r,
  }
  profile /sbin/tune2fs (complain) {
    /sbin/tune2fs rm,

    /lib/** rm,

    /apparmor/.null w,

    /etc/blkid.conf r,
    /etc/mtab r,
    /etc/ld.so.cache r,

    /dev/null rw,
    /dev/.blkid.tab r,
    /dev/dm-* rw,

    /proc/swaps r,
    /proc/[0-9]*/mounts r,
  }
  profile /sbin/blkid (complain) {
    /sbin/blkid rm,

    /lib/** rm,
    /apparmor/.null w,

    /etc/ld.so.cache r,
    /etc/blkid.conf r,

    /dev/null rw,
    /dev/.blkid.tab rl,
    /dev/.blkid.tab* rwl,
    /dev/dm-* r,

    /sys/devices/virtual/block/** r,

    capability mknod,

    mount -> @{DOCKER_GRAPH_PATH}/**,
  }
  profile /sbin/apparmor_parser (complain) {
    /sbin/apparmor_parser rm,

    /lib/** rm,

    /etc/ld.so.cache r,
    /etc/apparmor/** r,
    /etc/apparmor.d/** r,
    /etc/apparmor.d/cache/** w,

    /dev/null rw,

    /sys/kernel/security/apparmor/** r,
    /sys/kernel/security/apparmor/.replace w,

    /proc/[0-9]*/mounts r,
    /proc/sys/kernel/osrelease r,
    /proc r,

    capability mac_admin,
  }
}`
<|endoftext|>


<commit_before>/**
 * Copyright 2014 @ S1N1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-11 21:33
 * description :
 * history :
 */
package ols

import (
	"errors"
	"fmt"
	"github.com/atnet/gof"
	"github.com/atnet/gof/web"
	"go2o/src/core/infrastructure/payment"
	"go2o/src/core/service/dps"
	"net/http"
	"strconv"
	"strings"
)

var aliPayObj *payment.AliPay = &payment.AliPay{
	Partner: "2088021187655650",
	Key:     "2b0y8466t9mh82s5ajptub2p3wgjzwmh",
	Seller:  "1515827759@qq.com",
}

type PaymentC struct {
	*BaseC
}

func getDomain(r *http.Request) string {
	var proto string
	if r.Proto == "HTTPS" {
		proto = "https://"
	} else {
		proto = "http://"
	}
	return proto + r.Host
}
func (this *PaymentC) Create(ctx *web.Context) {
	r, w := ctx.Request, ctx.Response
	qs := r.URL.Query()
	partnerId := this.GetPartnerId(ctx)
	orderNo := qs.Get("order_no")
	paymentOpt := qs.Get("pay_opt")

	if len(orderNo) != 0 {
		if paymentOpt == "alipay" {
			domain := getDomain(ctx.Request)
			returnUrl := fmt.Sprintf("%s/pay/return_alipay", domain)
			notifyUrl := fmt.Sprintf("%s/pay/notify/%d_alipay", domain, partnerId)
			gateway := aliPayObj.CreateGateway(orderNo, 0.01, "在线支付订单", "订单号："+orderNo, notifyUrl, returnUrl)
			html := "<meta charset=\"utf-8\"/>" + gateway + ""
			w.Write([]byte(html))

			payment.Debug(" [ Submit] - %s - %s", orderNo, notifyUrl)

			return
		}
	}

	w.Write([]byte("订单不存在"))
}

func (this *PaymentC) Return_alipay(ctx *web.Context) {
	result := aliPayObj.Return(ctx.Request)
	if result.Status == payment.StatusTradeSuccess {
		this.handleOrder(this.GetPartnerId(ctx), "alipay", &result)
		this.paymentSuccess(ctx, &result)
		return
	}
	this.paymentFail(ctx, &result)
}

func (this *PaymentC) Notify_post(ctx *web.Context) {
	path := ctx.Request.URL.Path
	lastSeg := strings.Split(path[strings.LastIndex(path, "/")+1:], "_")
	paymentOpt := lastSeg[1]
	partnerId, _ := strconv.Atoi(lastSeg[0])

	payment.Debug(" [ Notify] - URL - %s - %d -  %s", ctx.Request.RequestURI, partnerId, paymentOpt)

	if paymentOpt == "alipay" {
		result := aliPayObj.Notify(ctx.Request)
		if result.Status == payment.StatusTradeSuccess {
			this.handleOrder(partnerId, "alipay", &result)
			payment.Debug("payment ok")
			ctx.Response.Write([]byte("success"))
			return
		}
	}
	ctx.Response.Write([]byte("fail"))
}

func (this *PaymentC) handleOrder(partnerId int, sp string, result *payment.Result) error {
	order := dps.ShoppingService.GetOrderByNo(partnerId, result.OrderNo)
	if order != nil {
		//		if order.PayFee != result.Fee{
		//			return errors.New(fmt.Sprintf("error order fee %f / %f",order.PayFee,result.Fee))
		//		}
		if order.IsPaid == 1 {
			return errors.New("order has paid")
		}
		return dps.ShoppingService.PayForOrderOnlineTrade(partnerId, order.OrderNo, sp, result.TradeNo)
	}
	return errors.New("no such order")
}

func (this *PaymentC) paymentSuccess(ctx *web.Context, result *payment.Result) {
	p := this.GetPartner(ctx)
	siteConf := this.GetSiteConf(ctx)

	this.BaseC.ExecuteTemplate(ctx, gof.TemplateDataMap{
		"partner": p,
		"conf":    siteConf,
	},
		"views/shop/ols/{device}/payment_success.html",
		"views/shop/ols/{device}/inc/header.html",
		"views/shop/ols/{device}/inc/footer.html")
}

func (this *PaymentC) paymentFail(ctx *web.Context, result *payment.Result) {
	p := this.GetPartner(ctx)
	siteConf := this.GetSiteConf(ctx)

	this.BaseC.ExecuteTemplate(ctx, gof.TemplateDataMap{
		"partner": p,
		"conf":    siteConf,
	},
		"views/shop/ols/{device}/payment_fail.html",
		"views/shop/ols/{device}/inc/header.html",
		"views/shop/ols/{device}/inc/footer.html")
}
<commit_msg>refactor<commit_after>/**
 * Copyright 2014 @ S1N1 Team.
 * name :
 * author : jarryliu
 * date : 2014-02-11 21:33
 * description :
 * history :
 */
package ols

import (
	"errors"
	"fmt"
	"github.com/atnet/gof"
	"github.com/atnet/gof/web"
	"go2o/src/core/infrastructure/payment"
	"go2o/src/core/service/dps"
	"net/http"
	"strconv"
	"strings"
)

var aliPayObj *payment.AliPay = &payment.AliPay{
	Partner: "2088021187655650",
	Key:     "2b0y8466t9mh82s5ajptub2p3wgjzwmh",
	Seller:  "1515827759@qq.com",
}

type PaymentC struct {
	*BaseC
}

func getDomain(r *http.Request) string {
	var proto string
	if r.Proto == "HTTPS" {
		proto = "https://"
	} else {
		proto = "http://"
	}
	return proto + r.Host
}
func (this *PaymentC) Create(ctx *web.Context) {
	r, w := ctx.Request, ctx.Response
	qs := r.URL.Query()
	partnerId := this.GetPartnerId(ctx)
	orderNo := qs.Get("order_no")
	paymentOpt := qs.Get("pay_opt")

	if len(orderNo) != 0 {
		if paymentOpt == "alipay" {
			domain := getDomain(ctx.Request)
			returnUrl := fmt.Sprintf("%s/pay/return_alipay", domain)
			notifyUrl := fmt.Sprintf("%s/pay/notify/%d_alipay", domain, partnerId)
			gateway := aliPayObj.CreateGateway(orderNo, 0.01, "在线支付订单", "订单号："+orderNo, notifyUrl, returnUrl)
			html := "<meta charset=\"utf-8\"/>" + gateway + ""
			w.Write([]byte(html))

			payment.Debug(" [ Submit] - %s - %s", orderNo, notifyUrl)

			return
		}
	}

	w.Write([]byte("订单不存在"))
}

func (this *PaymentC) Return_alipay(ctx *web.Context) {
	//this.paymentFail(ctx,nil)
	//return
	result := aliPayObj.Return(ctx.Request)
	if result.Status == payment.StatusTradeSuccess {
		this.handleOrder(this.GetPartnerId(ctx), "alipay", &result)
		this.paymentSuccess(ctx, &result)
		return
	}
	this.paymentFail(ctx, &result)
}

func (this *PaymentC) Notify_post(ctx *web.Context) {
	path := ctx.Request.URL.Path
	lastSeg := strings.Split(path[strings.LastIndex(path, "/")+1:], "_")
	paymentOpt := lastSeg[1]
	partnerId, _ := strconv.Atoi(lastSeg[0])

	payment.Debug(" [ Notify] - URL - %s - %d -  %s", ctx.Request.RequestURI, partnerId, paymentOpt)

	if paymentOpt == "alipay" {
		result := aliPayObj.Notify(ctx.Request)
		if result.Status == payment.StatusTradeSuccess {
			this.handleOrder(partnerId, "alipay", &result)
			payment.Debug("payment ok")
			ctx.Response.Write([]byte("success"))
			return
		}
	}
	ctx.Response.Write([]byte("fail"))
}

func (this *PaymentC) handleOrder(partnerId int, sp string, result *payment.Result) error {
	order := dps.ShoppingService.GetOrderByNo(partnerId, result.OrderNo)
	if order != nil {
		//		if order.PayFee != result.Fee{
		//			return errors.New(fmt.Sprintf("error order fee %f / %f",order.PayFee,result.Fee))
		//		}
		if order.IsPaid == 1 {
			return errors.New("order has paid")
		}
		return dps.ShoppingService.PayForOrderOnlineTrade(partnerId, order.OrderNo, sp, result.TradeNo)
	}
	return errors.New("no such order")
}

func (this *PaymentC) paymentSuccess(ctx *web.Context, result *payment.Result) {
	p := this.GetPartner(ctx)
	siteConf := this.GetSiteConf(ctx)

	this.BaseC.ExecuteTemplate(ctx, gof.TemplateDataMap{
		"partner": p,
		"conf":    siteConf,
	},
		"views/shop/ols/{device}/payment_success.html",
		"views/shop/ols/{device}/inc/header.html",
		"views/shop/ols/{device}/inc/footer.html")
}

func (this *PaymentC) paymentFail(ctx *web.Context, result *payment.Result) {
	p := this.GetPartner(ctx)
	siteConf := this.GetSiteConf(ctx)

	this.BaseC.ExecuteTemplate(ctx, gof.TemplateDataMap{
		"partner": p,
		"conf":    siteConf,
	},
		"views/shop/ols/{device}/payment_fail.html",
		"views/shop/ols/{device}/inc/header.html",
		"views/shop/ols/{device}/inc/footer.html")
}
<|endoftext|>


<commit_before>package js // import "github.com/tdewolff/parse/js"

var regexpStateByte = [128]bool{
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false, //
	true,  // !
	false, // "
	false, // #
	false, // $
	false, // %
	true,  // &
	false, // '
	true,  // (
	false, // )
	true,  // *
	true,  // +
	true,  // ,
	true,  // -
	false, // .
	true,  // /
	false, // 0
	false, // 1
	false, // 2
	false, // 3
	false, // 4
	false, // 5
	false, // 6
	false, // 7
	false, // 8
	false, // 9
	true,  // :
	true,  // ;
	false, // <
	true,  // =
	false, // >
	true,  // ?
	false, // @
	false, // A-Z...
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	true,  // [
	false, // \
	false, // ]
	false, // ^
	false, // _
	false, // `
	false, // a-z...
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	false,
	true, // {
	true, // |
	true, // }
	true, // ~
	false,
}
<commit_msg>Remove now-unused regex state table<commit_after><|endoftext|>


<commit_before>package consensus

import (
	"container/heap"
	"container/vector"
	"goprotobuf.googlecode.com/hg/proto"
)

type packet struct {
	Addr string
	M
}

func (p packet) Less(y interface{}) bool {
	return *p.Seqn < *y.(packet).Seqn
}

type Packet struct {
	Addr string
	Data []byte
}

type Stats struct {
	Runs        int
	WaitPackets int
}

type Manager <-chan Stats

type Prop struct {
	Seqn int64
	Mut  []byte
}

func NewManager(self string, propSeqns chan<- int64, in <-chan Packet, runs <-chan *run, props <-chan *Prop) Manager {
	statCh := make(chan Stats)
	propRuns := make(chan *run)

	go filterPropSeqns(self, propRuns, propSeqns)

	go func() {
		running := make(map[int64]*run)
		packets := new(vector.Vector)
		ticks := make(chan int64)
		var nextRun int64
		var stats Stats

		for {
			stats.Runs = len(running)
			stats.WaitPackets = packets.Len()

			select {
			case run := <-runs:
				if closed(runs) {
					return
				}

				running[run.seqn] = run
				nextRun = run.seqn + 1
				run.ticks = ticks
				propRuns <- run
			case p := <-in:
				recvPacket(packets, p)
			case n := <-ticks:
				schedTick(packets, n)
			case statCh <- stats:
			case pr := <-props:
				m := M{Seqn: &pr.Seqn, Cmd: propose, Value: pr.Mut}
				heap.Push(packets, packet{M: m})
			}

			for packets.Len() > 0 {
				p := packets.At(0).(packet)

				seqn := *p.Seqn

				if seqn >= nextRun {
					break
				}

				heap.Pop(packets)

				running[seqn].deliver(p)
			}
		}
	}()

	return statCh
}

func filterPropSeqns(id string, rc <-chan *run, sc chan<- int64) {
	for r := range rc {
		if r.isLeader(id) {
			sc <- r.seqn
		}
	}
}

func recvPacket(q heap.Interface, P Packet) {
	var p packet

	err := proto.Unmarshal(P.Data, &p.M)
	if err != nil {
		return
	}

	if p.M.Seqn == nil {
		return
	}

	heap.Push(q, p)
}

func schedTick(q heap.Interface, n int64) {
	heap.Push(q, packet{M: M{Seqn: proto.Int64(n), Cmd: tick}})
}
<commit_msg>consensus: document our contract more fully<commit_after>package consensus

import (
	"container/heap"
	"container/vector"
	"goprotobuf.googlecode.com/hg/proto"
)

type packet struct {
	Addr string
	M
}

func (p packet) Less(y interface{}) bool {
	return *p.Seqn < *y.(packet).Seqn
}

type Packet struct {
	Addr string
	Data []byte
}

type Stats struct {
	Runs        int
	WaitPackets int
}

type Manager <-chan Stats

type Prop struct {
	Seqn int64
	Mut  []byte
}

// propSeqns must be buffered with capacity >= alpha
func NewManager(self string, propSeqns chan<- int64, in <-chan Packet, runs <-chan *run, props <-chan *Prop) Manager {
	statCh := make(chan Stats)
	propRuns := make(chan *run)

	go filterPropSeqns(self, propRuns, propSeqns)

	go func() {
		running := make(map[int64]*run)
		packets := new(vector.Vector)
		ticks := make(chan int64)
		var nextRun int64
		var stats Stats

		for {
			stats.Runs = len(running)
			stats.WaitPackets = packets.Len()

			select {
			case run := <-runs:
				if closed(runs) {
					return
				}

				running[run.seqn] = run
				nextRun = run.seqn + 1
				run.ticks = ticks
				propRuns <- run
			case p := <-in:
				recvPacket(packets, p)
			case n := <-ticks:
				schedTick(packets, n)
			case statCh <- stats:
			case pr := <-props:
				m := M{Seqn: &pr.Seqn, Cmd: propose, Value: pr.Mut}
				heap.Push(packets, packet{M: m})
			}

			for packets.Len() > 0 {
				p := packets.At(0).(packet)

				seqn := *p.Seqn

				if seqn >= nextRun {
					break
				}

				heap.Pop(packets)

				running[seqn].deliver(p)
			}
		}
	}()

	return statCh
}

func filterPropSeqns(id string, rc <-chan *run, sc chan<- int64) {
	for r := range rc {
		if r.isLeader(id) {
			sc <- r.seqn
		}
	}
}

func recvPacket(q heap.Interface, P Packet) {
	var p packet

	err := proto.Unmarshal(P.Data, &p.M)
	if err != nil {
		return
	}

	if p.M.Seqn == nil {
		return
	}

	heap.Push(q, p)
}

func schedTick(q heap.Interface, n int64) {
	heap.Push(q, packet{M: M{Seqn: proto.Int64(n), Cmd: tick}})
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"testing"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Logf("skipping during short test")
		return
	}
	runtime.GOMAXPROCS(3)
	compl := make(chan int, 1)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- 0
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
	}()
	go perpetuumMobile()
	<-compl
	stop <- true
}
<commit_msg>runtime: reset GOMAXPROCS during tests<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"runtime"
	"testing"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Logf("skipping during short test")
		return
	}
	maxprocs := runtime.GOMAXPROCS(3)
	compl := make(chan bool, 2)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- true
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
		compl <- true
	}()
	go perpetuumMobile()
	<-compl
	<-compl
	stop <- true
	runtime.GOMAXPROCS(maxprocs)
}
<|endoftext|>


<commit_before>package peer

import (
	"fmt"
	"io/ioutil"
	"testing"
)

func TestCrStates(t *testing.T) {
	t.Log("Running Peer Tests")

	text, err := ioutil.ReadFile("crstates.json")
	if err != nil {
		t.Log(err)
	}
	crStates, err := CrStatesUnMarshall(text)
	if err != nil {
		t.Log(err)
	}
	fmt.Println(len(crStates.Caches), "caches found")
	for cacheName, crState := range crStates.Caches {
		t.Logf("%v -> %v", cacheName, crState.IsAvailable)
	}

	fmt.Println(len(crStates.Deliveryservice), "deliveryservices found")
	for dsName, deliveryService := range crStates.Deliveryservice {
		t.Logf("%v -> %v (len:%v)", dsName, deliveryService.IsAvailable, len(deliveryService.DisabledLocations))
	}

}
<commit_msg>Fix TM2 test to match renamed symbol<commit_after>package peer

import (
	"fmt"
	"io/ioutil"
	"testing"
)

func TestCrStates(t *testing.T) {
	t.Log("Running Peer Tests")

	text, err := ioutil.ReadFile("crstates.json")
	if err != nil {
		t.Log(err)
	}
	crStates, err := CrstatesUnMarshall(text)
	if err != nil {
		t.Log(err)
	}
	fmt.Println(len(crStates.Caches), "caches found")
	for cacheName, crState := range crStates.Caches {
		t.Logf("%v -> %v", cacheName, crState.IsAvailable)
	}

	fmt.Println(len(crStates.Deliveryservice), "deliveryservices found")
	for dsName, deliveryService := range crStates.Deliveryservice {
		t.Logf("%v -> %v (len:%v)", dsName, deliveryService.IsAvailable, len(deliveryService.DisabledLocations))
	}

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"github.com/SocialCodeInc/go-gelf/gelf"
	"io"
	"log"
	"os"
	"os/exec"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor						string `json:"__CURSOR"`
	Realtime_timestamp			int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp			string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id						string `json:"_BOOT_ID"`
	Transport					string `json:"_TRANSPORT"`
	Priority					int32  `json:"PRIORITY,string"`
	Syslog_facility				string `json:"SYSLOG_FACILITY"`
	Syslog_identifier			string `json:"SYSLOG_IDENTIFIER"`
	Message						string `json:"MESSAGE"`
	Pid							string `json:"_PID"`
	Uid							string `json:"_UID"`
	Gid							string `json:"_GID"`
	Comm						string `json:"_COMM"`
	Exe							string `json:"_EXE"`
	Cmdline						string `json:"_CMDLINE"`
	Systemd_cgroup				string `json:"_SYSTEMD_CGROUP"`
	Systemd_session				string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid			string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit				string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp	string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id					string `json:"_MACHINE_ID"`
	Hostname					string `json:"_HOSTNAME"`
	FullMessage					string
}

func (this *SystemdJournalEntry) toGelf() (*gelf.Message, error) {
	return &gelf.Message{
		Version: 	"1.0",
		Host:		this.Hostname,
		Short:		this.Message,
		Full:		this.FullMessage,
		TimeUnix:	this.Realtime_timestamp / 1000 / 1000,
		Level:		this.Priority,
		Facility:	this.Syslog_identifier,
		Extra: map[string]interface{}{
			"Boot_id":	this.Boot_id,
			"Pid":		this.Pid,
			"Uid":		this.Uid,
		},
	}, nil
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	// less than 1 second in between
	if this.Realtime_timestamp-message.Realtime_timestamp > 1000000 {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	if message, err := this.toGelf(); err != nil {
		log.Print(err)
	} else {
		go gelfWriter.WriteMessage(message)
		pendingEntry = nil
	}
}

var (
	pendingEntry *SystemdJournalEntry
	gelfWriter   *gelf.Writer
)

func main() {
	if len(os.Args) < 3 {
		log.Fatal("Pass server:12201 as first argument and append journalctl parameters to use")
	}

	serverAddr := os.Args[1]
	journalArgs := os.Args[1:]
	journalArgs[0] = "--output=json"

	cmd := exec.Command("journalctl", journalArgs...)

	var err error
	gelfWriter, err = gelf.NewWriter(serverAddr)
	if err != nil {
		log.Fatalf("While connecting to Graylog server: %s", err)
	}

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)

	go writePendingEntry()

	// Larger buffer for systemd's inline coredumps which are typically ~ 14Kb
	r := bufio.NewReaderSize(stdout, 16384)
	cmd.Start()

	for line, _, err := r.ReadLine(); err != io.EOF; line, _, err = r.ReadLine() {
		if err != nil {
			break
		}

		entry := new(SystemdJournalEntry)

		if err = json.Unmarshal(line, &entry); err != nil {
			log.Printf("Could not parse line, skipping: %s\n", line)
			continue
		}

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) {
			pendingEntry.send()
			pendingEntry = entry
		} else {
			if pendingEntry.FullMessage == "" {
				pendingEntry.FullMessage = pendingEntry.Message
			}

			entry.FullMessage = pendingEntry.FullMessage + "\n" + entry.Message
			entry.Message = pendingEntry.Message

			// Replace pending so the higher timestamp keeps writePendingEntry waiting longer for us to append even more
			pendingEntry = entry
		}
	}

	cmd.Wait()
}

func writePendingEntry() {
	for {
		time.Sleep(100 * time.Millisecond)

		if pendingEntry != nil {
			pendingEntry.send()
		}
	}
}<commit_msg>fixed a bug, a pending should not be send every 100ms regardless its timestamp<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"github.com/SocialCodeInc/go-gelf/gelf"
	"io"
	"log"
	"os"
	"os/exec"
	"time"
)

/*
	http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
	https://github.com/Graylog2/graylog2-docs/wiki/GELF
*/
type SystemdJournalEntry struct {
	Cursor						string `json:"__CURSOR"`
	Realtime_timestamp			int64  `json:"__REALTIME_TIMESTAMP,string"`
	Monotonic_timestamp			string `json:"__MONOTONIC_TIMESTAMP"`
	Boot_id						string `json:"_BOOT_ID"`
	Transport					string `json:"_TRANSPORT"`
	Priority					int32  `json:"PRIORITY,string"`
	Syslog_facility				string `json:"SYSLOG_FACILITY"`
	Syslog_identifier			string `json:"SYSLOG_IDENTIFIER"`
	Message						string `json:"MESSAGE"`
	Pid							string `json:"_PID"`
	Uid							string `json:"_UID"`
	Gid							string `json:"_GID"`
	Comm						string `json:"_COMM"`
	Exe							string `json:"_EXE"`
	Cmdline						string `json:"_CMDLINE"`
	Systemd_cgroup				string `json:"_SYSTEMD_CGROUP"`
	Systemd_session				string `json:"_SYSTEMD_SESSION"`
	Systemd_owner_uid			string `json:"_SYSTEMD_OWNER_UID"`
	Systemd_unit				string `json:"_SYSTEMD_UNIT"`
	Source_realtime_timestamp	string `json:"_SOURCE_REALTIME_TIMESTAMP"`
	Machine_id					string `json:"_MACHINE_ID"`
	Hostname					string `json:"_HOSTNAME"`
	FullMessage					string
}

func (this *SystemdJournalEntry) toGelf() (*gelf.Message, error) {
	return &gelf.Message{
		Version: 	"1.0",
		Host:		this.Hostname,
		Short:		this.Message,
		Full:		this.FullMessage,
		TimeUnix:	this.Realtime_timestamp / 1000 / 1000,
		Level:		this.Priority,
		Facility:	this.Syslog_identifier,
		Extra: map[string]interface{}{
			"Boot_id":	this.Boot_id,
			"Pid":		this.Pid,
			"Uid":		this.Uid,
		},
	}, nil
}

func (this *SystemdJournalEntry) sameSource(message *SystemdJournalEntry) bool {
	if this.Syslog_identifier != message.Syslog_identifier {
		return false
	}

	if this.Priority != message.Priority {
		return false
	}

	// less than 1 second in between
	if this.Realtime_timestamp-message.Realtime_timestamp > 1000000 {
		return false
	}

	return true
}

func (this *SystemdJournalEntry) send() {
	if message, err := this.toGelf(); err != nil {
		log.Print(err)
	} else {
		go gelfWriter.WriteMessage(message)
		pendingEntry = nil
	}
}

var (
	pendingEntry *SystemdJournalEntry
	gelfWriter   *gelf.Writer
)

func main() {
	if len(os.Args) < 3 {
		log.Fatal("Pass server:12201 as first argument and append journalctl parameters to use")
	}

	serverAddr := os.Args[1]
	journalArgs := os.Args[1:]
	journalArgs[0] = "--output=json"

	cmd := exec.Command("journalctl", journalArgs...)

	var err error
	gelfWriter, err = gelf.NewWriter(serverAddr)
	if err != nil {
		log.Fatalf("While connecting to Graylog server: %s", err)
	}

	stdout, _ := cmd.StdoutPipe()
	stderr, _ := cmd.StderrPipe()
	go io.Copy(os.Stderr, stderr)

	go writePendingEntry()

	// Larger buffer for systemd's inline coredumps which are typically ~ 14Kb
	r := bufio.NewReaderSize(stdout, 16384)
	cmd.Start()

	for line, _, err := r.ReadLine(); err != io.EOF; line, _, err = r.ReadLine() {
		if err != nil {
			break
		}

		entry := new(SystemdJournalEntry)

		if err = json.Unmarshal(line, &entry); err != nil {
			log.Printf("Could not parse line, skipping: %s\n", line)
			continue
		}

		if pendingEntry == nil {
			pendingEntry = entry
		} else if !pendingEntry.sameSource(entry) {
			pendingEntry.send()
			pendingEntry = entry
		} else {
			if pendingEntry.FullMessage == "" {
				pendingEntry.FullMessage = pendingEntry.Message
			}

			entry.FullMessage = pendingEntry.FullMessage + "\n" + entry.Message
			entry.Message = pendingEntry.Message

			// Replace pending so the higher timestamp keeps writePendingEntry waiting longer for us to append even more
			pendingEntry = entry
		}
	}

	cmd.Wait()
}

func writePendingEntry() {
	for {
		time.Sleep(50 * time.Millisecond)

		if pendingEntry != nil && (time.Now().UnixNano() / 1000 - pendingEntry.Realtime_timestamp) > 100 * 1000 {
			pendingEntry.send()
		}
	}
}<|endoftext|>


<commit_before>package controllerlib

import (
	"math"
)

type UpdateRequest struct {
	MyName 	string
	Type	string
	XPos	int
	YPos	int
}

var XSize = 25
var YSize = 25

func GetGridCode(r UpdateRequest) string {
	switch r.Type {
		case "Empty":
			return " "
		case "Robber":
			return "R"
		case "Cop":
			return "C"
		default:
			return "?"
	}
}

func WhereAmI(GameState [][]UpdateRequest, me UpdateRequest) (int, int) {
        for i := range GameState {
                for j := range GameState[i] {
			//fmt.Println("Comparing: ", me.MyName, GameState[i][j].MyName)
			if me.MyName == GameState[i][j].MyName {
				return GameState[i][j].XPos, GameState[i][j].YPos
			}
                }
        }
	return -1, -1
}

func WhereNearest(GameState [][]UpdateRequest, me UpdateRequest, mytype string) (UpdateRequest) {
	nearest := UpdateRequest{"Empty", mytype, 0, 0}
	nearestDistance := XSize
        for i := range GameState {
                for j := range GameState[i] {
			if mytype == GameState[i][j].Type {
				distance = CalculateDistance(me, GameState[i][j])
				if distance < nearestDistance {
					nearestDistance = distance
					nearest = GameState[i][j]
			}
                }
        }
	return nearest
}

func CalculateDistance(me UpdateRequest, them UpdateRequest) (int) {
	xDistance = math.Abs(them.XPos - me.XPos)
	yDistance = math.Abs(them.YPos - me.YPos)
	distance = math.Sqrt(math.Pow(xDistance, 2) + math.Pow(yDistance, 2))
	return int(distance)
}
<commit_msg>syntax<commit_after>package controllerlib

import (
	"math"
)

type UpdateRequest struct {
	MyName 	string
	Type	string
	XPos	int
	YPos	int
}

var XSize = 25
var YSize = 25

func GetGridCode(r UpdateRequest) string {
	switch r.Type {
		case "Empty":
			return " "
		case "Robber":
			return "R"
		case "Cop":
			return "C"
		default:
			return "?"
	}
}

func WhereAmI(GameState [][]UpdateRequest, me UpdateRequest) (int, int) {
        for i := range GameState {
                for j := range GameState[i] {
			//fmt.Println("Comparing: ", me.MyName, GameState[i][j].MyName)
			if me.MyName == GameState[i][j].MyName {
				return GameState[i][j].XPos, GameState[i][j].YPos
			}
                }
        }
	return -1, -1
}

func WhereNearest(GameState [][]UpdateRequest, me UpdateRequest, mytype string) (UpdateRequest) {
	nearest := UpdateRequest{"Empty", mytype, 0, 0}
	nearestDistance := XSize
        for i := range GameState {
                for j := range GameState[i] {
			if mytype == GameState[i][j].Type {
				distance = CalculateDistance(me, GameState[i][j])
				if distance < nearestDistance {
					nearestDistance = distance
					nearest = GameState[i][j]
				}
			}
                }
        }
	return nearest
}

func CalculateDistance(me UpdateRequest, them UpdateRequest) (int) {
	xDistance = math.Abs(them.XPos - me.XPos)
	yDistance = math.Abs(them.YPos - me.YPos)
	distance = math.Sqrt(math.Pow(xDistance, 2) + math.Pow(yDistance, 2))
	return int(distance)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long the node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 10*time.Minute,
		"How often scale down possiblity is check")
	scanInterval = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	unneededNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						glog.V(2).Info("Schedulable pods present")
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					unneededStart := time.Now()

					// In dry run only utilization is updated
					calculateUnneededOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialInterval).After(time.Now()) ||
						schedulablePodsPresent

					glog.V(4).Info("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
						"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
						lastScaleUpTime, lastScaleDownFailedTrial, schedulablePodsPresent)

					updateLastTime("findUnneeded")
					glog.V(4).Infof("Calculating unneded nodes")

					unneededNodes = FindUnneededNodes(
						nodes,
						unneededNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("findUnneeded", unneededStart)

					for key, val := range unneededNodes {
						if glog.V(4) {
							glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
						}
					}

					if !calculateUnneededOnly {
						glog.V(4).Infof("Starting scale down")

						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							unneededNodes,
							*scaleDownUnneededTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the map with unneeded nodes to be super sure that the simulated
								// deletions are made in the new context.
								unneededNodes = make(map[string]time.Time, len(unneededNodes))
							} else {
								lastScaleDownFailedTrial = time.Now()
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<commit_msg>Cluster-autoscaler: timing in scale down fix<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"net/http"
	"net/url"
	"os"
	"time"

	"k8s.io/contrib/cluster-autoscaler/config"
	"k8s.io/contrib/cluster-autoscaler/simulator"
	"k8s.io/contrib/cluster-autoscaler/utils/gce"
	kube_api "k8s.io/kubernetes/pkg/api"
	kube_record "k8s.io/kubernetes/pkg/client/record"
	kube_client "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	migConfigFlag           config.MigConfigFlag
	address                 = flag.String("address", ":8085", "The address to expose prometheus metrics.")
	kubernetes              = flag.String("kubernetes", "", "Kuberentes master location. Leave blank for default")
	cloudConfig             = flag.String("cloud-config", "", "The path to the cloud provider configuration file.  Empty string for no configuration file.")
	verifyUnschedulablePods = flag.Bool("verify-unschedulable-pods", true,
		"If enabled CA will ensure that each pod marked by Scheduler as unschedulable actually can't be scheduled on any node."+
			"This prevents from adding unnecessary nodes in situation when CA and Scheduler have different configuration.")
	scaleDownEnabled = flag.Bool("scale-down-enabled", true, "Should CA scale down the cluster")
	scaleDownDelay   = flag.Duration("scale-down-delay", 10*time.Minute,
		"Duration from the last scale up to the time when CA starts to check scale down options")
	scaleDownUnneededTime = flag.Duration("scale-down-unneeded-time", 10*time.Minute,
		"How long the node should be unneeded before it is eligible for scale down")
	scaleDownUtilizationThreshold = flag.Float64("scale-down-utilization-threshold", 0.5,
		"Node utilization level, defined as sum of requested resources divided by capacity, below which a node can be considered for scale down")
	scaleDownTrialInterval = flag.Duration("scale-down-trial-interval", 1*time.Minute,
		"How often scale down possiblity is check")
	scanInterval = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
)

func main() {
	flag.Var(&migConfigFlag, "nodes", "sets min,max size and url of a MIG to be controlled by Cluster Autoscaler. "+
		"Can be used multiple times. Format: <min>:<max>:<migurl>")
	flag.Parse()

	go func() {
		http.Handle("/metrics", prometheus.Handler())
		err := http.ListenAndServe(*address, nil)
		glog.Fatalf("Failed to start metrics: %v", err)
	}()

	url, err := url.Parse(*kubernetes)
	if err != nil {
		glog.Fatalf("Failed to parse Kuberentes url: %v", err)
	}

	// Configuration
	kubeConfig, err := config.GetKubeClientConfig(url)
	if err != nil {
		glog.Fatalf("Failed to build Kuberentes client configuration: %v", err)
	}
	migConfigs := make([]*config.MigConfig, 0, len(migConfigFlag))
	for i := range migConfigFlag {
		migConfigs = append(migConfigs, &migConfigFlag[i])
	}

	// GCE Manager
	var gceManager *gce.GceManager
	if *cloudConfig != "" {
		config, err := os.Open(*cloudConfig)
		if err != nil {
			glog.Fatalf("Couldn't open cloud provider configuration %s: %#v", *cloudConfig, err)
		}
		defer config.Close()
		gceManager, err = gce.CreateGceManager(migConfigs, config)
	} else {
		gceManager, err = gce.CreateGceManager(migConfigs, nil)
	}
	if err != nil {
		glog.Fatalf("Failed to create GCE Manager: %v", err)
	}

	kubeClient := kube_client.NewOrDie(kubeConfig)

	predicateChecker, err := simulator.NewPredicateChecker(kubeClient)
	if err != nil {
		glog.Fatalf("Failed to create predicate checker: %v", err)
	}
	unschedulablePodLister := NewUnschedulablePodLister(kubeClient)
	scheduledPodLister := NewScheduledPodLister(kubeClient)
	nodeLister := NewNodeLister(kubeClient)

	lastScaleUpTime := time.Now()
	lastScaleDownFailedTrial := time.Now()
	unneededNodes := make(map[string]time.Time)

	eventBroadcaster := kube_record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(kubeClient.Events(""))
	recorder := eventBroadcaster.NewRecorder(kube_api.EventSource{Component: "cluster-autoscaler"})

	for {
		select {
		case <-time.After(*scanInterval):
			{
				loopStart := time.Now()
				updateLastTime("main")

				nodes, err := nodeLister.List()
				if err != nil {
					glog.Errorf("Failed to list nodes: %v", err)
					continue
				}
				if len(nodes) == 0 {
					glog.Errorf("No nodes in the cluster")
					continue
				}

				if err := CheckMigsAndNodes(nodes, gceManager); err != nil {
					glog.Warningf("Cluster is not ready for autoscaling: %v", err)
					continue
				}

				allUnschedulablePods, err := unschedulablePodLister.List()
				if err != nil {
					glog.Errorf("Failed to list unscheduled pods: %v", err)
					continue
				}

				allScheduled, err := scheduledPodLister.List()
				if err != nil {
					glog.Errorf("Failed to list scheduled pods: %v", err)
					continue
				}

				// We need to reset all pods that have been marked as unschedulable not after
				// the newest node became available for the scheduler.
				allNodesAvailableTime := GetAllNodesAvailableTime(nodes)
				podsToReset, unschedulablePodsToHelp := SlicePodsByPodScheduledTime(allUnschedulablePods, allNodesAvailableTime)
				ResetPodScheduledCondition(kubeClient, podsToReset)

				// We need to check whether pods marked as unschedulable are actually unschedulable.
				// This should prevent from adding unnecessary nodes. Example of such situation:
				// - CA and Scheduler has slightly different configuration
				// - Scheduler can't schedule a pod and marks it as unschedulable
				// - CA added a node which should help the pod
				// - Scheduler doesn't schedule the pod on the new node
				//   because according to it logic it doesn't fit there
				// - CA see the pod is still unschedulable, so it adds another node to help it
				//
				// With the check enabled the last point won't happen because CA will ignore a pod
				// which is supposed to schedule on an existing node.
				//
				// Without below check cluster might be unnecessary scaled up to the max allowed size
				// in the describe situation.
				schedulablePodsPresent := false
				if *verifyUnschedulablePods {
					newUnschedulablePodsToHelp := FilterOutSchedulable(unschedulablePodsToHelp, nodes, allScheduled, predicateChecker)

					if len(newUnschedulablePodsToHelp) != len(unschedulablePodsToHelp) {
						glog.V(2).Info("Schedulable pods present")
						schedulablePodsPresent = true
					}
					unschedulablePodsToHelp = newUnschedulablePodsToHelp
				}

				if len(unschedulablePodsToHelp) == 0 {
					glog.V(1).Info("No unschedulable pods")
				} else {
					scaleUpStart := time.Now()
					updateLastTime("scaleup")
					scaledUp, err := ScaleUp(unschedulablePodsToHelp, nodes, migConfigs, gceManager, kubeClient, predicateChecker, recorder)

					updateDuration("scaleup", scaleUpStart)

					if err != nil {
						glog.Errorf("Failed to scale up: %v", err)
						continue
					} else {
						if scaledUp {
							lastScaleUpTime = time.Now()
							// No scale down in this iteration.
							continue
						}
					}
				}

				if *scaleDownEnabled {
					unneededStart := time.Now()

					// In dry run only utilization is updated
					calculateUnneededOnly := lastScaleUpTime.Add(*scaleDownDelay).After(time.Now()) ||
						lastScaleDownFailedTrial.Add(*scaleDownTrialInterval).After(time.Now()) ||
						schedulablePodsPresent

					glog.V(4).Infof("Scale down status: unneededOnly=%v lastScaleUpTime=%s "+
						"lastScaleDownFailedTrail=%s schedulablePodsPresent=%v", calculateUnneededOnly,
						lastScaleUpTime, lastScaleDownFailedTrial, schedulablePodsPresent)

					updateLastTime("findUnneeded")
					glog.V(4).Infof("Calculating unneded nodes")

					unneededNodes = FindUnneededNodes(
						nodes,
						unneededNodes,
						*scaleDownUtilizationThreshold,
						allScheduled,
						predicateChecker)

					updateDuration("findUnneeded", unneededStart)

					for key, val := range unneededNodes {
						if glog.V(4) {
							glog.V(4).Infof("%s is unneeded since %s duration %s", key, val.String(), time.Now().Sub(val).String())
						}
					}

					if !calculateUnneededOnly {
						glog.V(4).Infof("Starting scale down")

						scaleDownStart := time.Now()
						updateLastTime("scaledown")

						result, err := ScaleDown(
							nodes,
							unneededNodes,
							*scaleDownUnneededTime,
							allScheduled,
							gceManager, kubeClient, predicateChecker)

						updateDuration("scaledown", scaleDownStart)

						// TODO: revisit result handling
						if err != nil {
							glog.Errorf("Failed to scale down: %v", err)
						} else {
							if result == ScaleDownNodeDeleted {
								// Clean the map with unneeded nodes to be super sure that the simulated
								// deletions are made in the new context.
								unneededNodes = make(map[string]time.Time, len(unneededNodes))
							} else {
								if result == ScaleDownError || result == ScaleDownNoNodeDeleted {
									lastScaleDownFailedTrial = time.Now()
								}
							}
						}
					}
				}
				updateDuration("main", loopStart)
			}
		}
	}
}

func updateDuration(label string, start time.Time) {
	duration.WithLabelValues(label).Observe(durationToMicro(start))
	lastDuration.WithLabelValues(label).Set(durationToMicro(start))
}

func updateLastTime(label string) {
	lastTimestamp.WithLabelValues(label).Set(float64(time.Now().Unix()))
}
<|endoftext|>