<commit_before>package engine

import (
	"fmt"
	"log"
	"time"

	"github.com/coreos/coreinit/job"
	"github.com/coreos/coreinit/machine"
	"github.com/coreos/coreinit/registry"
)

const (
	DefaultJobWatchClaimTTL = "10s"
	DefaultRefreshInterval  = "60s"
)

type JobWatcher struct {
	registry        *registry.Registry
	scheduler       *Scheduler
	machine         *machine.Machine
	claimTTL        time.Duration
	refreshInterval time.Duration
	watches         map[string]job.JobWatch
	schedules       map[string]Schedule
	machines        map[string]machine.Machine
}

func NewJobWatcher(reg *registry.Registry, scheduler *Scheduler, m *machine.Machine) *JobWatcher {
	claimTTL, _ := time.ParseDuration(DefaultJobWatchClaimTTL)
	refreshInterval, _ := time.ParseDuration(DefaultRefreshInterval)

	jobs := make(map[string]job.JobWatch, 0)
	schedules := make(map[string]Schedule, 0)
	machines := make(map[string]machine.Machine, 0)

	return &JobWatcher{reg, scheduler, m, claimTTL, refreshInterval, jobs, schedules, machines}
}

func (self *JobWatcher) StartHeartbeatThread() {
	heartbeat := func() {
		for _, watch := range self.watches {
			self.registry.ClaimJobWatch(&watch, self.machine, self.claimTTL)
		}
	}

	loop := func() {
		for true {
			heartbeat()
			time.Sleep(self.claimTTL / 2)
		}
	}

	go loop()
}

func (self *JobWatcher) StartRefreshThread() {
	refresh := func() {
		machines := make(map[string]machine.Machine, 0)
		for _, m := range self.registry.GetActiveMachines() {
			machines[m.BootId] = m
		}
		self.machines = machines
	}

	loop := func() {
		for true {
			refresh()
			time.Sleep(self.refreshInterval)
		}
	}

	go loop()
}

func (self *JobWatcher) AddJobWatch(watch *job.JobWatch) bool {
	if !self.registry.ClaimJobWatch(watch, self.machine, self.claimTTL) {
		return false
	}

	self.watches[watch.Payload.Name] = *watch
	sched := NewSchedule()
	self.schedules[watch.Payload.Name] = sched

	if watch.Count == -1 {
		for _, m := range self.machines {
			name := fmt.Sprintf("%s.%s", m.BootId, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)
			log.Printf("EventJobWatchCreated(%s): adding to schedule job=%s machine=%s", watch.Payload.Name, name, m.BootId)
			sched.Add(*j, m)
		}
	} else {
		for i := 1; i <= watch.Count; i++ {
			m := pickRandomMachine(self.machines)
			name := fmt.Sprintf("%d.%s", i, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)
			log.Printf("EventJobWatchCreated(%s): adding to schedule job=%s machine=%s", watch.Payload.Name, name, m.BootId)
			sched.Add(*j, *m)
		}
	}

	if len(sched) > 0 {
		log.Printf("EventJobWatchCreated(%s): submitting schedule", watch.Payload.Name)
		self.submitSchedule(sched)
	} else {
		log.Printf("EventJobWatchCreated(%s): no schedule changes made", watch.Payload.Name)
	}

	return true
}

func (self *JobWatcher) RemoveJobWatch(watch *job.JobWatch) bool {
	if _, ok := self.watches[watch.Payload.Name]; !ok {
		return false
	}

	delete(self.watches, watch.Payload.Name)

	watchSchedule := self.schedules[watch.Payload.Name]
	delete(self.schedules, watch.Payload.Name)

	for job, mach := range watchSchedule {
		self.registry.RemoveMachineJob(&job, mach)
	}

	return true
}

func (self *JobWatcher) submitSchedule(schedule Schedule) {
	for j, m := range schedule {
		self.registry.ScheduleMachineJob(&j, m)
	}
}

func (self *JobWatcher) TrackMachine(m *machine.Machine) {
	self.machines[m.BootId] = *m

	partial := NewSchedule()
	for _, watch := range self.watches {
		if watch.Count == -1 {
			name := fmt.Sprintf("%s.%s", m.BootId, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)
			log.Printf("Adding to schedule job=%s machine=%s", name, m.BootId)
			partial.Add(*j, *m)

			sched := self.schedules[watch.Payload.Name]
			sched.Add(*j, *m)
		}
	}

	if len(partial) > 0 {
		log.Printf("Submitting schedule")
		self.submitSchedule(partial)
	} else {
		log.Printf("No schedule changes made")
	}
}

func (self *JobWatcher) DropMachine(m *machine.Machine) {
	if _, ok := self.machines[m.BootId]; ok {
		delete(self.machines, m.BootId)
	}
}
<commit_msg>fix(JobWatcher): find machines running jobs before attempting to reschedule<commit_after>package engine

import (
	"fmt"
	"log"
	"time"

	"github.com/coreos/coreinit/job"
	"github.com/coreos/coreinit/machine"
	"github.com/coreos/coreinit/registry"
)

const (
	DefaultJobWatchClaimTTL = "10s"
	DefaultRefreshInterval  = "60s"
)

type JobWatcher struct {
	registry        *registry.Registry
	scheduler       *Scheduler
	machine         *machine.Machine
	claimTTL        time.Duration
	refreshInterval time.Duration
	watches         map[string]job.JobWatch
	schedules       map[string]Schedule
	machines        map[string]machine.Machine
}

func NewJobWatcher(reg *registry.Registry, scheduler *Scheduler, m *machine.Machine) *JobWatcher {
	claimTTL, _ := time.ParseDuration(DefaultJobWatchClaimTTL)
	refreshInterval, _ := time.ParseDuration(DefaultRefreshInterval)

	jobs := make(map[string]job.JobWatch, 0)
	schedules := make(map[string]Schedule, 0)
	machines := make(map[string]machine.Machine, 0)

	return &JobWatcher{reg, scheduler, m, claimTTL, refreshInterval, jobs, schedules, machines}
}

func (self *JobWatcher) StartHeartbeatThread() {
	heartbeat := func() {
		for _, watch := range self.watches {
			self.registry.ClaimJobWatch(&watch, self.machine, self.claimTTL)
		}
	}

	loop := func() {
		for true {
			heartbeat()
			time.Sleep(self.claimTTL / 2)
		}
	}

	go loop()
}

func (self *JobWatcher) StartRefreshThread() {
	refresh := func() {
		machines := make(map[string]machine.Machine, 0)
		for _, m := range self.registry.GetActiveMachines() {
			machines[m.BootId] = m
		}
		self.machines = machines
	}

	loop := func() {
		for true {
			refresh()
			time.Sleep(self.refreshInterval)
		}
	}

	go loop()
}

func (self *JobWatcher) AddJobWatch(watch *job.JobWatch) bool {
	if !self.registry.ClaimJobWatch(watch, self.machine, self.claimTTL) {
		return false
	}

	self.watches[watch.Payload.Name] = *watch
	sched := NewSchedule()
	self.schedules[watch.Payload.Name] = sched

	if watch.Count == -1 {
		for _, m := range self.machines {
			name := fmt.Sprintf("%s.%s", m.BootId, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)
			log.Printf("EventJobWatchCreated(%s): adding to schedule job=%s machine=%s", watch.Payload.Name, name, m.BootId)
			sched.Add(*j, m)
		}
	} else {
		for i := 1; i <= watch.Count; i++ {
			name := fmt.Sprintf("%d.%s", i, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)

			var m *machine.Machine
			// Check if this job was schedule somewhere already
			if state := self.registry.GetJobState(j); state != nil {
				log.Printf("Found job already schedule to machine")
				m = state.Machine
			} else {
				m = pickRandomMachine(self.machines)
			}

			log.Printf("EventJobWatchCreated(%s): adding to schedule job=%s machine=%s", watch.Payload.Name, name, m.BootId)
			sched.Add(*j, *m)
		}
	}

	if len(sched) > 0 {
		log.Printf("EventJobWatchCreated(%s): submitting schedule", watch.Payload.Name)
		self.submitSchedule(sched)
	} else {
		log.Printf("EventJobWatchCreated(%s): no schedule changes made", watch.Payload.Name)
	}

	return true
}

func (self *JobWatcher) RemoveJobWatch(watch *job.JobWatch) bool {
	if _, ok := self.watches[watch.Payload.Name]; !ok {
		return false
	}

	delete(self.watches, watch.Payload.Name)

	watchSchedule := self.schedules[watch.Payload.Name]
	delete(self.schedules, watch.Payload.Name)

	for job, mach := range watchSchedule {
		self.registry.RemoveMachineJob(&job, mach)
	}

	return true
}

func (self *JobWatcher) submitSchedule(schedule Schedule) {
	for j, m := range schedule {
		self.registry.ScheduleMachineJob(&j, m)
	}
}

func (self *JobWatcher) TrackMachine(m *machine.Machine) {
	self.machines[m.BootId] = *m

	partial := NewSchedule()
	for _, watch := range self.watches {
		if watch.Count == -1 {
			name := fmt.Sprintf("%s.%s", m.BootId, watch.Payload.Name)
			j, _ := job.NewJob(name, nil, watch.Payload)
			log.Printf("Adding to schedule job=%s machine=%s", name, m.BootId)
			partial.Add(*j, *m)

			sched := self.schedules[watch.Payload.Name]
			sched.Add(*j, *m)
		}
	}

	if len(partial) > 0 {
		log.Printf("Submitting schedule")
		self.submitSchedule(partial)
	} else {
		log.Printf("No schedule changes made")
	}
}

func (self *JobWatcher) DropMachine(m *machine.Machine) {
	if _, ok := self.machines[m.BootId]; ok {
		delete(self.machines, m.BootId)
	}
}
<|endoftext|>


<commit_before>package espsdk

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
)

// Serializable objects can be Marshaled into JSON.
type Serializable interface {
	Marshal() ([]byte, error)
}

// A Client is able to request an access token and submit HTTP requests to
// the ESP API.
type Client struct {
	Credentials
	UploadBucket string
}

// GetToken submits the provided credentials to Getty's OAuth2 endpoint
// and returns a token that can be used to authenticate HTTP requests to the
// ESP API.
func (c Client) GetToken() Token {
	if c.Credentials.areInvalid() {
		log.Fatal("Not all required credentials were supplied.")
	}

	uri := oauthEndpoint
	log.Debugf("%s", uri)
	formValues := c.formValues()
	log.Debugf("%s", formValues.Encode())

	resp, err := http.PostForm(uri, formValues)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	payload, err := ioutil.ReadAll(resp.Body)
	log.Debugf("HTTP %d", resp.StatusCode)
	log.Debugf("%s", payload)
	return c.tokenFrom(payload)
}

func (c Client) tokenFrom(payload []byte) Token {
	var response map[string]string
	json.Unmarshal(payload, &response)
	return Token(response["access_token"])
}

// PerformRequest performs a request using the given parameters and
// returns a struct that contains the HTTP status code and payload from
// the server's response as well as metadata such as the response time.
func (c Client) PerformRequest(p *request) *FulfilledRequest {
	uri := ESPAPIRoot + p.Path

	if p.requiresAnObject() && p.Object != nil {
		log.Debugf("Received serialized object: %s", p.Object)
	}
	req, err := http.NewRequest(p.Verb, uri, bytes.NewBuffer(p.Object))
	if err != nil {
		log.Fatal(err)
	}
	p.httpRequest = req

	p.addHeaders(p.Token, c.APIKey)

	result := getResult(insecureClient(), req)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	return &FulfilledRequest{p, result}
}

func (c *Client) get(path string) []byte {
	request := newRequest("GET", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) newGet(path string) []byte {
	request := newRequest("GET", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) post(object interface{}, path string) []byte {
	serializedObject, err := Marshal(object)
	if err != nil {
		log.Fatal(err)
	}

	request := newRequest("POST", path, c.GetToken(), serializedObject)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) put(object Serializable, path string) []byte {
	serializedObject, err := object.Marshal()
	if err != nil {
		log.Fatal(err)
	}

	request := newRequest("PUT", path, c.GetToken(), serializedObject)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) _delete(path string) {
	request := newRequest("DELETE", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
}

// insecureClient returns an HTTP client that will not verify the validity
// of an SSL certificate when performing a request.
func insecureClient() *http.Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	return &http.Client{Transport: tr}
}
<commit_msg>create Client.Delete() method<commit_after>package espsdk

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"io/ioutil"
	"net/http"

	log "github.com/Sirupsen/logrus"
)

// Serializable objects can be Marshaled into JSON.
type Serializable interface {
	Marshal() ([]byte, error)
}

// A Client is able to request an access token and submit HTTP requests to
// the ESP API.
type Client struct {
	Credentials
	UploadBucket string
}

// GetToken submits the provided credentials to Getty's OAuth2 endpoint
// and returns a token that can be used to authenticate HTTP requests to the
// ESP API.
func (c Client) GetToken() Token {
	if c.Credentials.areInvalid() {
		log.Fatal("Not all required credentials were supplied.")
	}

	uri := oauthEndpoint
	log.Debugf("%s", uri)
	formValues := c.formValues()
	log.Debugf("%s", formValues.Encode())

	resp, err := http.PostForm(uri, formValues)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	payload, err := ioutil.ReadAll(resp.Body)
	log.Debugf("HTTP %d", resp.StatusCode)
	log.Debugf("%s", payload)
	return c.tokenFrom(payload)
}

func (c Client) tokenFrom(payload []byte) Token {
	var response map[string]string
	json.Unmarshal(payload, &response)
	return Token(response["access_token"])
}

// PerformRequest performs a request using the given parameters and
// returns a struct that contains the HTTP status code and payload from
// the server's response as well as metadata such as the response time.
func (c Client) PerformRequest(p *request) *FulfilledRequest {
	uri := ESPAPIRoot + p.Path

	if p.requiresAnObject() && p.Object != nil {
		log.Debugf("Received serialized object: %s", p.Object)
	}
	req, err := http.NewRequest(p.Verb, uri, bytes.NewBuffer(p.Object))
	if err != nil {
		log.Fatal(err)
	}
	p.httpRequest = req

	p.addHeaders(p.Token, c.APIKey)

	result := getResult(insecureClient(), req)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	return &FulfilledRequest{p, result}
}

// Delete destroys the object at the provided path.
func (c *Client) Delete(path string) { c._delete(path) }

func (c *Client) get(path string) []byte {
	request := newRequest("GET", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) newGet(path string) []byte {
	request := newRequest("GET", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}
	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) post(object interface{}, path string) []byte {
	serializedObject, err := Marshal(object)
	if err != nil {
		log.Fatal(err)
	}

	request := newRequest("POST", path, c.GetToken(), serializedObject)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) put(object Serializable, path string) []byte {
	serializedObject, err := object.Marshal()
	if err != nil {
		log.Fatal(err)
	}

	request := newRequest("PUT", path, c.GetToken(), serializedObject)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
	return result.Payload
}

func (c *Client) _delete(path string) {
	request := newRequest("DELETE", path, c.GetToken(), nil)
	result := c.PerformRequest(request)
	if result.Err != nil {
		log.Fatal(result.Err)
	}

	stats, err := result.Marshal()
	if err != nil {
		log.Fatal(result.Err)
	}
	log.Info(string(stats))
	log.Debugf("%s\n", result.Payload)
}

// insecureClient returns an HTTP client that will not verify the validity
// of an SSL certificate when performing a request.
func insecureClient() *http.Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	return &http.Client{Transport: tr}
}
<|endoftext|>


<commit_before>package bsdatomtoics

import (
  "fmt"
  "net/http"
  "io"
  "io/ioutil"
  "os"
  "encoding/xml"
  "strings"
  "time"
)

const queryURI = "https://www.beaverton.k12.or.us/_vti_bin/BSD.Extranet/Syndication.svc/GetDistrictCalendarFeed?format=atom"
const outputfmt = "%s\n"

// Atom Feed Data Structure

type Feed struct {
  XMLName xml.Name "http://www.w3.org/2005/Atom feed"
  Title   string   `xml:"title"`
  Id      string   `xml:"id"`
  Link    []Link   `xml:"link"`
  Updated Time     `xml:"updated"`
  Entry   []Entry  `xml:"entry"`
}

type Entry struct {
  Title   string `xml:"title"`
  Id      string `xml:"id"`
  Link    []Link `xml:"link"`
  Updated Time   `xml:"updated"`
  Content string `xml:"content"`
}

type Link struct {
  Rel    string   "attr"
  Href   string   "attr"
}

type Text struct {
  Type   string   "attr"
  Body   string   "chardata"
}

type Time string

func FetchBytes() ([]byte, error) {
  return FetchBytesWith(http.DefaultClient)
}

func FetchBytesWith(client *http.Client) ([]byte, error) {
  r, err := client.Get(queryURI)
  if (err != nil || r.StatusCode != 200) {
     return nil, err
  }
  rc, err := ioutil.ReadAll(r.Body)
  defer r.Body.Close()
  if (err != nil) { return nil, err }
  return rc, nil
}

func AtomToICS(bytes []byte, writer io.Writer, debug bool) {
  var bsd Feed

  if (bytes == nil || len(bytes) == 0) {
    fmt.Fprintf(writer, "no bytes received in input to AtomToICS")
    return
  }
  err := xml.Unmarshal(bytes, &bsd)

  if (debug){
    fmt.Fprintf(os.Stderr, "Title: %s\n", bsd.Title)
    fmt.Fprintf(os.Stderr, "Id: %s\n", bsd.Id)
    fmt.Fprintf(os.Stderr, "Last updated: %v\n", len(bsd.Updated))
    fmt.Fprintf(os.Stderr, "Entry count after unmarshal: %v\n", len(bsd.Entry))
  }
  if err == nil {
    fmt.Fprintf(writer, "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//BSDATOMTOICS v1.0//EN\r\n")
    for i := 0; i < len(bsd.Entry); i++ {
      fmt.Fprintf(writer, "BEGIN:VEVENT\r\n")
      fmt.Fprintf(writer, "SUMMARY:%s\r\n", bsd.Entry[i].Title)
      start, end, location := parseStartEndLocation(bsd.Entry[i].Content, debug)
      fmt.Fprintf(writer, "DTSTART:%s\r\n", start)
      fmt.Fprintf(writer, "DTEND:%s\r\n", end)
      fmt.Fprintf(writer, "LOCATION:%s\r\n", location)
      fmt.Fprintf(writer, "END:VEVENT\r\n")
    }
    fmt.Fprintf(writer, "END:VCALENDAR\r\n")
  } else {
    fmt.Fprintf(os.Stderr, "Unable to parse the Atom feed (%v)\n", err)
  }
}

func parseStartEndLocation(content string, debug bool) (string, string, string) {
  //Event Time: 3/23/2015 12:00:00 PM - 3/27/2015 1:00:00 PM  Location:  Spring Break - Schools closed
  //fmt.Fprintf(os.Stderr, "Raw input: '%s'\n", content)
  strippedContent := strings.Replace(content, "\n", "", -1)
  reallyStrippedContent := strings.Replace(strippedContent, "\r", "", -1)
  eventTimeRemoved := strings.TrimLeft(strings.Replace(reallyStrippedContent, "Event Time: ", "", 1), " ")
  if (debug) {
    fmt.Fprintf(os.Stderr, "After removal: '%s'\n", eventTimeRemoved)
  }
  timeFromLocation := strings.SplitAfterN(eventTimeRemoved, "Location: ", 2)
  time := timeFromLocation[0]
  location := timeFromLocation[1]
  startFromEnd := strings.SplitAfterN(time, " - ", 2)
  return toUTC(strings.Replace(startFromEnd[0], " - ", "", 1)),
         toUTC(strings.TrimRight(strings.Replace(startFromEnd[1], "Location: ", "", 1), " ")),
         strings.Trim(location, " ")
}

func toUTC(timeStr string) string {
  loc, _ := time.LoadLocation("America/Los_Angeles")
  //fmt.Fprintf(os.Stderr, "Input: |%s|\n", timeStr)
  //Mon Jan 2 15:04:05 -0700 MST 2006
  shortForm := "1/2/2006 3:04:05 PM"
  timeVal, _ := time.ParseInLocation(shortForm, timeStr, loc)
  return timeVal.UTC().Format("20060102T030400Z")
}<commit_msg>support school events feeds<commit_after>package bsdatomtoics

import (
  "fmt"
  "net/http"
  "io"
  "io/ioutil"
  "os"
  "encoding/xml"
  "strings"
  "time"
)

const queryStart = "https://www.beaverton.k12.or.us/"
const queryEnd = "_vti_bin/BSD.Extranet/Syndication.svc/"
//District: "https://www.beaverton.k12.or.us/_vti_bin/BSD.Extranet/Syndication.svc/GetDistrictCalendarFeed?format=atom"
//JW "https://www.beaverton.k12.or.us/schools/jacob-wismer/_vti_bin/BSD.Extranet/Syndication.svc/GetSchoolEventsFeed?format=atom"
const outputfmt = "%s\n"

// Atom Feed Data Structure

type Feed struct {
  XMLName xml.Name "http://www.w3.org/2005/Atom feed"
  Title   string   `xml:"title"`
  Id      string   `xml:"id"`
  Link    []Link   `xml:"link"`
  Updated Time     `xml:"updated"`
  Entry   []Entry  `xml:"entry"`
}

type Entry struct {
  Title   string `xml:"title"`
  Id      string `xml:"id"`
  Link    []Link `xml:"link"`
  Updated Time   `xml:"updated"`
  Content string `xml:"content"`
}

type Link struct {
  Rel    string   "attr"
  Href   string   "attr"
}

type Text struct {
  Type   string   "attr"
  Body   string   "chardata"
}

type Time string

func FetchBytes() ([]byte, error) {
  return FetchBytesWith(http.DefaultClient, "")
}

func FetchBytesWith(client *http.Client, school string) ([]byte, error) {
   finalUri := queryStart;
   if school != "" { //jacob-wismer
      finalUri = finalUri + "schools/" + school + "/"
   }
   finalUri = finalUri + queryEnd
   if school == "" {
      finalUri = finalUri + "GetDistrictCalendarFeed?format=atom"
   } else {
      finalUri = finalUri + "GetSchoolEventsFeed?format=atom"
   }

   r, err := client.Get(finalUri)
   if (err != nil || r.StatusCode != 200) {
      return nil, err
   }
   rc, err := ioutil.ReadAll(r.Body);
   defer r.Body.Close();
   if (err != nil) { return nil, err }
   return rc, nil;
}

func AtomToICS(bytes []byte, writer io.Writer, debug bool) {
  var bsd Feed

  if (bytes == nil || len(bytes) == 0) {
    fmt.Fprintf(writer, "no bytes received in input to AtomToICS")
    return
  }
  err := xml.Unmarshal(bytes, &bsd)

  if (debug){
    fmt.Fprintf(os.Stderr, "Title: %s\n", bsd.Title)
    fmt.Fprintf(os.Stderr, "Id: %s\n", bsd.Id)
    fmt.Fprintf(os.Stderr, "Last updated: %v\n", len(bsd.Updated))
    fmt.Fprintf(os.Stderr, "Entry count after unmarshal: %v\n", len(bsd.Entry))
  }
  if err == nil {
    fmt.Fprintf(writer, "BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//BSDATOMTOICS v1.0//EN\r\n")
    for i := 0; i < len(bsd.Entry); i++ {
      fmt.Fprintf(writer, "BEGIN:VEVENT\r\n")
      fmt.Fprintf(writer, "SUMMARY:%s\r\n", bsd.Entry[i].Title)
      start, end, location := parseStartEndLocation(bsd.Entry[i].Content, debug)
      fmt.Fprintf(writer, "DTSTART:%s\r\n", start)
      fmt.Fprintf(writer, "DTEND:%s\r\n", end)
      fmt.Fprintf(writer, "LOCATION:%s\r\n", location)
      fmt.Fprintf(writer, "END:VEVENT\r\n")
    }
    fmt.Fprintf(writer, "END:VCALENDAR\r\n")
  } else {
    fmt.Fprintf(os.Stderr, "Unable to parse the Atom feed (%v)\n", err)
  }
}

func parseStartEndLocation(content string, debug bool) (string, string, string) {
  //Event Time: 3/23/2015 12:00:00 PM - 3/27/2015 1:00:00 PM  Location:  Spring Break - Schools closed
  //fmt.Fprintf(os.Stderr, "Raw input: '%s'\n", content)
  strippedContent := strings.Replace(content, "\n", "", -1)
  reallyStrippedContent := strings.Replace(strippedContent, "\r", "", -1)
  eventTimeRemoved := strings.TrimLeft(strings.Replace(reallyStrippedContent, "Event Time: ", "", 1), " ")
  if (debug) {
    fmt.Fprintf(os.Stderr, "After removal: '%s'\n", eventTimeRemoved)
  }
  timeFromLocation := strings.SplitAfterN(eventTimeRemoved, "Location: ", 2)
  time := timeFromLocation[0]
  location := timeFromLocation[1]
  startFromEnd := strings.SplitAfterN(time, " - ", 2)
  return toUTC(strings.Replace(startFromEnd[0], " - ", "", 1)),
         toUTC(strings.TrimRight(strings.Replace(startFromEnd[1], "Location: ", "", 1), " ")),
         strings.Trim(location, " ")
}

func toUTC(timeStr string) string {
  loc, _ := time.LoadLocation("America/Los_Angeles")
  //fmt.Fprintf(os.Stderr, "Input: |%s|\n", timeStr)
  //Mon Jan 2 15:04:05 -0700 MST 2006
  shortForm := "1/2/2006 3:04:05 PM"
  timeVal, _ := time.ParseInLocation(shortForm, timeStr, loc)
  return timeVal.UTC().Format("20060102T030400Z")
}<|endoftext|>


<commit_before>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package ast

import (
	"bytes"
	"fmt"
	"strings"
)

// The Node interface is implemented by all nodes in the AST.
type Node interface {
	Type() NodeType
	String() string
	Pos() *PosInfo
	Len() int
	Offset(offset int) *PosInfo
}

// The NodeType data type describes the type of a Node.
type NodeType int

const (
	ErrorType   NodeType = iota // ErrorType is the default type, not an actual node type.
	FileType                    // FileType contains text or comment nodes
	TextType                    // TextType contains text
	CommentType                 // CommentType contains a comment
)

func (t NodeType) String() string {
	switch t {
	case ErrorType:
		return "error"
	case FileType:
		return "file"
	case TextType:
		return "text"
	case CommentType:
		return "comment"
	default:
		return "unknown"
	}
}

// PosInfo {{{

// The PosInfo data type describes text positions in the original file.
type PosInfo struct {
	Name   string
	Line   int
	Column int
}

// Pos returns itself, useful for composition.
func (p PosInfo) Pos() *PosInfo { return &p }

// String returns the standard string representation of position information:
//
//  name:line:column
//
func (p PosInfo) String() string {
	return fmt.Sprintf("%s:%d:%d", p.Name, p.Line, p.Column)
}

func (p PosInfo) OffsetIn(data string, offset int) *PosInfo {
	if offset > len(data) {
		return nil
	}
	code := data[:offset]
	pi := &PosInfo{
		Name: p.Name,
		Line: p.Line + strings.Count(code, "\n"),
	}
	if i := strings.LastIndex(code, "\n"); i >= 0 {
		pi.Column = offset - i
	} else {
		pi.Column = 1 + len(code)
	}
	return pi
}

// }}}

// TextNode {{{

type TextNode struct {
	PosInfo
	val string
}

func (n TextNode) Type() NodeType             { return TextType }
func (n TextNode) String() string             { return n.val }
func (n TextNode) Len() int                   { return len(n.val) }
func (n TextNode) Offset(offset int) *PosInfo { return n.OffsetIn(n.val, offset) }

// }}}

// CommentNode {{{

type CommentNode struct {
	PosInfo
	val string
	c   *Commenter
}

func (n CommentNode) Type() NodeType             { return CommentType }
func (n CommentNode) String() string             { return n.val }
func (n CommentNode) Len() int                   { return len(n.val) }
func (n CommentNode) Offset(offset int) *PosInfo { return n.OffsetIn(n.val, offset) }

// }}}

// FileNode {{{

type FileNode struct {
	PosInfo
	name  string
	path  string
	root  *FileNode
	nodes []Node
}

func (fn FileNode) Type() NodeType { return FileType }

func (fn FileNode) String() string {
	var buf bytes.Buffer
	for _, n := range fn.nodes {
		buf.WriteString(n.String())
	}
	return buf.String()
}

func (fn FileNode) Len() int {
	var total int
	for _, n := range fn.nodes {
		total += n.Len()
	}
	return total
}

func (fn FileNode) Offset(offset int) *PosInfo {
	for _, n := range fn.nodes {
		pi := n.Offset(offset)
		if pi != nil {
			return pi
		}
		offset -= n.Len()
	}
	return nil
}

func (fn FileNode) Nodes() []Node {
	var nodes []Node
	for _, n := range fn.nodes {
		if n.Type() == FileType {
			nodes = append(nodes, n.(*FileNode).Nodes()...)
			continue
		}
		nodes = append(nodes, n)
	}
	return nodes
}

func (fn *FileNode) addNode(n Node) {
	fn.nodes = append(fn.nodes, n)
}

// }}}
<commit_msg>Adding OffsetLC to enable better pos.info. retrieval<commit_after>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package ast

import (
	"bytes"
	"fmt"
	"strings"
)

// The Node interface is implemented by all nodes in the AST.
type Node interface {
	Type() NodeType
	String() string
	Pos() *PosInfo
	Len() int
	Offset(offset int) *PosInfo
	OffsetLC(line, col int) *PosInfo
}

// The NodeType data type describes the type of a Node.
type NodeType int

const (
	ErrorType   NodeType = iota // ErrorType is the default type, not an actual node type.
	FileType                    // FileType contains text or comment nodes
	TextType                    // TextType contains text
	CommentType                 // CommentType contains a comment
)

func (t NodeType) String() string {
	switch t {
	case ErrorType:
		return "error"
	case FileType:
		return "file"
	case TextType:
		return "text"
	case CommentType:
		return "comment"
	default:
		return "unknown"
	}
}

// PosInfo {{{

// The PosInfo data type describes text positions in the original file.
type PosInfo struct {
	Name   string
	Line   int
	Column int
}

// Pos returns itself, useful for composition.
func (p PosInfo) Pos() *PosInfo { return &p }

// String returns the standard string representation of position information:
//
//  name:line:column
//
func (p PosInfo) String() string {
	return fmt.Sprintf("%s:%d:%d", p.Name, p.Line, p.Column)
}

func (p PosInfo) OffsetIn(data string, offset int) *PosInfo {
	if offset > len(data) {
		return nil
	}
	code := data[:offset]
	pi := &PosInfo{
		Name: p.Name,
		Line: p.Line + strings.Count(code, "\n"),
	}
	if i := strings.LastIndex(code, "\n"); i >= 0 {
		pi.Column = offset - i
	} else {
		pi.Column = 1 + len(code)
	}
	return pi
}

func (p PosInfo) OffsetInLC(data string, line, col int) *PosInfo {
	line, col = line-1, col-1
	if strings.Count(data, "\n") <= line {
		return nil
	}

	return &PosInfo{
		Name:   p.Name,
		Line:   p.Line + line,
		Column: p.Column + col,
	}
}

// }}}

// TextNode {{{

type TextNode struct {
	PosInfo
	val string
}

func (n TextNode) Type() NodeType                  { return TextType }
func (n TextNode) String() string                  { return n.val }
func (n TextNode) Len() int                        { return len(n.val) }
func (n TextNode) Offset(offset int) *PosInfo      { return n.OffsetIn(n.val, offset) }
func (n TextNode) OffsetLC(line, col int) *PosInfo { return n.OffsetInLC(n.val, line, col) }

// }}}

// CommentNode {{{

type CommentNode struct {
	PosInfo
	val string
	c   *Commenter
}

func (n CommentNode) Type() NodeType                  { return CommentType }
func (n CommentNode) String() string                  { return n.val }
func (n CommentNode) Len() int                        { return len(n.val) }
func (n CommentNode) Offset(offset int) *PosInfo      { return n.OffsetIn(n.val, offset) }
func (n CommentNode) OffsetLC(line, col int) *PosInfo { return n.OffsetInLC(n.val, line, col) }

// }}}

// FileNode {{{

type FileNode struct {
	PosInfo
	name  string
	path  string
	root  *FileNode
	nodes []Node
}

func (fn FileNode) Type() NodeType { return FileType }

func (fn FileNode) String() string {
	var buf bytes.Buffer
	for _, n := range fn.nodes {
		buf.WriteString(n.String())
	}
	return buf.String()
}

func (fn FileNode) Len() int {
	var total int
	for _, n := range fn.nodes {
		total += n.Len()
	}
	return total
}

func (fn FileNode) OffsetLC(line, col int) *PosInfo {
	for _, n := range fn.nodes {
		pi := n.OffsetLC(line, col)
		if pi != nil {
			return pi
		}
		// TODO: make this more efficient!
		line -= strings.Count(n.String(), "\n")
	}
	return nil
}

func (fn FileNode) Offset(offset int) *PosInfo {
	for _, n := range fn.nodes {
		pi := n.Offset(offset)
		if pi != nil {
			return pi
		}
		offset -= n.Len()
	}
	return nil
}

func (fn FileNode) Nodes() []Node {
	var nodes []Node
	for _, n := range fn.nodes {
		if n.Type() == FileType {
			nodes = append(nodes, n.(*FileNode).Nodes()...)
			continue
		}
		nodes = append(nodes, n)
	}
	return nodes
}

func (fn *FileNode) addNode(n Node) {
	fn.nodes = append(fn.nodes, n)
}

// }}}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"strings"

	"k8s.io/api/core/v1"
	utilnet "k8s.io/utils/net"
)

/*
This file is duplicated from "k8s.io/kubernetes/pkg/api/v1/service/util.go"
in order for in-tree cloud providers to not depend on internal packages.
*/

const (
	defaultLoadBalancerSourceRanges = "0.0.0.0/0"
)

// IsAllowAll checks whether the utilnet.IPNet allows traffic from 0.0.0.0/0
func IsAllowAll(ipnets utilnet.IPNetSet) bool {
	for _, s := range ipnets.StringSlice() {
		if s == "0.0.0.0/0" {
			return true
		}
	}
	return false
}

// GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service.
// If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service,
// extracting the source ranges to allow, and if not present returns a default (allow-all) value.
func GetLoadBalancerSourceRanges(service *v1.Service) (utilnet.IPNetSet, error) {
	var ipnets utilnet.IPNetSet
	var err error
	// if SourceRange field is specified, ignore sourceRange annotation
	if len(service.Spec.LoadBalancerSourceRanges) > 0 {
		specs := service.Spec.LoadBalancerSourceRanges
		ipnets, err = utilnet.ParseIPNets(specs...)

		if err != nil {
			return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err)
		}
	} else {
		val := service.Annotations[v1.AnnotationLoadBalancerSourceRangesKey]
		val = strings.TrimSpace(val)
		if val == "" {
			val = defaultLoadBalancerSourceRanges
		}
		specs := strings.Split(val, ",")
		ipnets, err = utilnet.ParseIPNets(specs...)
		if err != nil {
			return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", v1.AnnotationLoadBalancerSourceRangesKey, val)
		}
	}
	return ipnets, nil
}

// GetServiceHealthCheckPathPort returns the path and nodePort programmed into the Cloud LB Health Check
func GetServiceHealthCheckPathPort(service *v1.Service) (string, int32) {
	if !NeedsHealthCheck(service) {
		return "", 0
	}
	port := service.Spec.HealthCheckNodePort
	if port == 0 {
		return "", 0
	}
	return "/healthz", port
}

// RequestsOnlyLocalTraffic checks if service requests OnlyLocal traffic.
func RequestsOnlyLocalTraffic(service *v1.Service) bool {
	if service.Spec.Type != v1.ServiceTypeLoadBalancer &&
		service.Spec.Type != v1.ServiceTypeNodePort {
		return false
	}
	return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
}

// NeedsHealthCheck checks if service needs health check.
func NeedsHealthCheck(service *v1.Service) bool {
	if service.Spec.Type != v1.ServiceTypeLoadBalancer {
		return false
	}
	return RequestsOnlyLocalTraffic(service)
}
<commit_msg>Add Load Balancer finalizer support<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"strings"

	v1 "k8s.io/api/core/v1"
	utilnet "k8s.io/utils/net"
)

/*
This file is duplicated from "k8s.io/kubernetes/pkg/api/v1/service/util.go"
in order for in-tree cloud providers to not depend on internal packages.
*/

const (
	defaultLoadBalancerSourceRanges = "0.0.0.0/0"

	// LoadBalancerCleanupFinalizer is the finalizer added to load balancer
	// services to ensure the Service resource is not fully deleted until
	// the correlating load balancer resources are deleted.
	LoadBalancerCleanupFinalizer = "service.kubernetes.io/load-balancer-cleanup"
)

// IsAllowAll checks whether the utilnet.IPNet allows traffic from 0.0.0.0/0
func IsAllowAll(ipnets utilnet.IPNetSet) bool {
	for _, s := range ipnets.StringSlice() {
		if s == "0.0.0.0/0" {
			return true
		}
	}
	return false
}

// GetLoadBalancerSourceRanges first try to parse and verify LoadBalancerSourceRanges field from a service.
// If the field is not specified, turn to parse and verify the AnnotationLoadBalancerSourceRangesKey annotation from a service,
// extracting the source ranges to allow, and if not present returns a default (allow-all) value.
func GetLoadBalancerSourceRanges(service *v1.Service) (utilnet.IPNetSet, error) {
	var ipnets utilnet.IPNetSet
	var err error
	// if SourceRange field is specified, ignore sourceRange annotation
	if len(service.Spec.LoadBalancerSourceRanges) > 0 {
		specs := service.Spec.LoadBalancerSourceRanges
		ipnets, err = utilnet.ParseIPNets(specs...)

		if err != nil {
			return nil, fmt.Errorf("service.Spec.LoadBalancerSourceRanges: %v is not valid. Expecting a list of IP ranges. For example, 10.0.0.0/24. Error msg: %v", specs, err)
		}
	} else {
		val := service.Annotations[v1.AnnotationLoadBalancerSourceRangesKey]
		val = strings.TrimSpace(val)
		if val == "" {
			val = defaultLoadBalancerSourceRanges
		}
		specs := strings.Split(val, ",")
		ipnets, err = utilnet.ParseIPNets(specs...)
		if err != nil {
			return nil, fmt.Errorf("%s: %s is not valid. Expecting a comma-separated list of source IP ranges. For example, 10.0.0.0/24,192.168.2.0/24", v1.AnnotationLoadBalancerSourceRangesKey, val)
		}
	}
	return ipnets, nil
}

// GetServiceHealthCheckPathPort returns the path and nodePort programmed into the Cloud LB Health Check
func GetServiceHealthCheckPathPort(service *v1.Service) (string, int32) {
	if !NeedsHealthCheck(service) {
		return "", 0
	}
	port := service.Spec.HealthCheckNodePort
	if port == 0 {
		return "", 0
	}
	return "/healthz", port
}

// RequestsOnlyLocalTraffic checks if service requests OnlyLocal traffic.
func RequestsOnlyLocalTraffic(service *v1.Service) bool {
	if service.Spec.Type != v1.ServiceTypeLoadBalancer &&
		service.Spec.Type != v1.ServiceTypeNodePort {
		return false
	}
	return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal
}

// NeedsHealthCheck checks if service needs health check.
func NeedsHealthCheck(service *v1.Service) bool {
	if service.Spec.Type != v1.ServiceTypeLoadBalancer {
		return false
	}
	return RequestsOnlyLocalTraffic(service)
}

// HasLBFinalizer checks if service contains LoadBalancerCleanupFinalizer.
func HasLBFinalizer(service *v1.Service) bool {
	for _, finalizer := range service.ObjectMeta.Finalizers {
		if finalizer == LoadBalancerCleanupFinalizer {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
)

type System interface {
	OS() string
	Arch() string
}

type DefaultSystem struct{}

func (ds DefaultSystem) OS() string {
	return runtime.GOOS
}

func (ds DefaultSystem) Arch() string {
	return runtime.GOARCH
}

type Logger interface {
	Debugf(string, ...interface{})
	Infof(string, ...interface{})
	Warnf(string, ...interface{})
}

type LogrusLogger struct{}

func (ll LogrusLogger) Debugf(str string, args ...interface{}) {
	logrus.Debugf(str, args...)
}

func (ll LogrusLogger) Infof(str string, args ...interface{}) {
	logrus.Infof(str, args...)
}

func (ll LogrusLogger) Warnf(str string, args ...interface{}) {
	logrus.Warnf(str, args...)
}

type Downloader interface {
	DownloadFile(string, string) error
	PullDockerImage(string) error
}

type DefaultDownloader struct {
	Logger
	Runner
}

func (dd DefaultDownloader) DownloadFile(url, path string) error {
	dd.Infof("Downloading file from %s to %s", url, path)

	res, err := http.Get(url)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("unable to download %s", url))
	}

	out, err := os.Create(path)
	defer out.Close()
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("unable to create file %s", path))
	}

	_, err = io.Copy(out, res.Body)
	if err != nil {
		return errors.Wrap(err, "unable to save downloaded file")
	}
	res.Body.Close()

	return nil
}

func (dd DefaultDownloader) PullDockerImage(image string) error {
	return dd.RunCommand("docker", []string{"pull", image})
}

type Runner interface {
	RunCommand(string, []string) error
	CheckCommand(string, []string) bool
}

type DefaultRunner struct {
	Logger
}

func (dr DefaultRunner) CheckCommand(command string, args []string) bool {
	dr.Infof("Checking command %s with args %v", command, args)

	return exec.Command(command, args...).Run() == nil
}

func (dr DefaultRunner) RunCommand(command string, args []string) error {
	dr.Infof("Running command %s with args %v", command, args)

	// adapted from https://gobyexample.com/execing-processes
	fullPath, err := exec.LookPath(command)
	if err != nil {
		return err
	}
	return syscall.Exec(fullPath, append([]string{command}, args...), os.Environ())
	// end adapted from
}
<commit_msg>set binary name when exec'ing<commit_after>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path"
	"runtime"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
)

type System interface {
	OS() string
	Arch() string
}

type DefaultSystem struct{}

func (ds DefaultSystem) OS() string {
	return runtime.GOOS
}

func (ds DefaultSystem) Arch() string {
	return runtime.GOARCH
}

type Logger interface {
	Debugf(string, ...interface{})
	Infof(string, ...interface{})
	Warnf(string, ...interface{})
}

type LogrusLogger struct{}

func (ll LogrusLogger) Debugf(str string, args ...interface{}) {
	logrus.Debugf(str, args...)
}

func (ll LogrusLogger) Infof(str string, args ...interface{}) {
	logrus.Infof(str, args...)
}

func (ll LogrusLogger) Warnf(str string, args ...interface{}) {
	logrus.Warnf(str, args...)
}

type Downloader interface {
	DownloadFile(string, string) error
	PullDockerImage(string) error
}

type DefaultDownloader struct {
	Logger
	Runner
}

func (dd DefaultDownloader) DownloadFile(url, path string) error {
	dd.Infof("Downloading file from %s to %s", url, path)

	res, err := http.Get(url)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("unable to download %s", url))
	}

	out, err := os.Create(path)
	defer out.Close()
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("unable to create file %s", path))
	}

	_, err = io.Copy(out, res.Body)
	if err != nil {
		return errors.Wrap(err, "unable to save downloaded file")
	}
	res.Body.Close()

	return nil
}

func (dd DefaultDownloader) PullDockerImage(image string) error {
	return dd.RunCommand("docker", []string{"pull", image})
}

type Runner interface {
	RunCommand(string, []string) error
	CheckCommand(string, []string) bool
}

type DefaultRunner struct {
	Logger
}

func (dr DefaultRunner) CheckCommand(command string, args []string) bool {
	dr.Infof("Checking command %s with args %v", command, args)

	return exec.Command(command, args...).Run() == nil
}

func (dr DefaultRunner) RunCommand(command string, args []string) error {
	dr.Infof("Running command %s with args %v", command, args)

	// adapted from https://gobyexample.com/execing-processes
	fullPath, err := exec.LookPath(command)
	if err != nil {
		return err
	}
	return syscall.Exec(fullPath, append([]string{path.Base(command)}, args...), os.Environ())
	// end adapted from
}
<|endoftext|>


<commit_before>package loda

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/lodastack/agent/agent/common"
	"github.com/lodastack/agent/agent/goplugin"
	"github.com/lodastack/agent/agent/plugins"
	"github.com/lodastack/log"
)

var Zerotimes = 0

func Get(url string) (b []byte, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("Get url failed: %s  not found", url)
		return
	}
	b, err = ioutil.ReadAll(resp.Body)
	return
}

func Post(url string, data []byte) ([]byte, error) {
	body := bytes.NewBuffer([]byte(data))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json;charset=utf-8")

	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = fmt.Errorf("Post url failed: %s code: %d", url, res.StatusCode)
		return nil, err
	}

	result, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func Ns() ([]string, error) {
	var res []string
	host, err := common.Hostname()
	if err != nil {
		return res, err
	}
	//check hostname chaged
	changed, _ := common.HostnameChanged()
	if changed {
		return res, fmt.Errorf("hostname changed, skip fetch ns")
	}

	url := fmt.Sprintf("%s/api/v1/agent/ns", common.Conf.RegistryAddr)
	data := make(map[string]string)
	data["hostname"] = host
	data["ip"] = strings.Join(common.GetIpList(), ",")
	byteData, err := json.Marshal(data)
	b, err := Post(url, byteData)
	if err != nil {
		return res, err
	}

	type ResponseNS struct {
		Code int               `json:"httpstatus"`
		Data map[string]string `json:"data"`
	}
	var response ResponseNS
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Warning("json.Marshal Ns failed: ", err)
		return res, err
	}

	resp := response.Data

	var ids []string
	for ns, id := range resp {
		res = append(res, ns)
		ids = append(ids, id)
	}
	common.SetUUID(ids)
	return res, nil
}

func pullResources(ns string) (res []map[string]string, err error) {
	url := fmt.Sprintf("%s/api/v1/agent/resource?ns=%s&type=collect", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}

	type ResponseRes struct {
		Code int                 `json:"httpstatus"`
		Data []map[string]string `json:"data"`
	}
	var response ResponseRes
	err = json.Unmarshal(b, &response)
	if err != nil {
		return
	}
	res = response.Data

	if len(res) == 0 {
		err = fmt.Errorf("no items under this namespace")
		Zerotimes++
	} else {
		Zerotimes = 0
	}
	return
}

func MonitorItems() (ports []common.PortMonitor,
	procs []common.ProcMonitor,
	pluginCollectors map[string]plugins.Collector,
	gopluginCollectors map[string]goplugin.Collector,
	intervals map[string]int, err error) {
	nss := common.GetNamespaces()
	pluginCollectors = make(map[string]plugins.Collector)
	pluginInfo := make(map[string]bool)
	gopluginCollectors = make(map[string]goplugin.Collector)
	intervals = make(map[string]int)

	for _, ns := range nss {
		err = nil
		var items []map[string]string
		items, err = pullResources(ns)
		if err != nil {
			log.Error("failed to get resources from registry, ns: ", ns, " err: ", err)
			continue
		}

		for _, item := range items {
			monitorType, ok := item["measurement_type"]
			if !ok {
				log.Warning("measurement_type is not exist: ", item["measurement_type"])
				continue
			}
			b, err := json.Marshal(item)
			if err != nil {
				log.Warning("json.Marshal item failed: ", err)
				continue
			}
			switch monitorType {
			case common.TYPE_PORT:
				if port, err := parsePort(b); err == nil {
					port.Namespace = ns
					ports = append(ports, port)
				} else {
					log.Warning("parsePort failed: ", err)
				}
			case common.TYPE_PROC:
				if proc, err := parseProc(b); err == nil {
					proc.Namespace = ns
					procs = append(procs, proc)
				} else {
					log.Warning("parseProc failed: ", err)
				}
			case common.TYPE_PLUGIN:
				if col, err := parsePlugin(b); err == nil {
					col.Namespace = ns
					pluginCollectors[ns+"|"+col.ProjectName] = col
					pluginInfo[ns+"|"+col.Repo] = true
				} else {
					log.Warning("get plugin collection failed: ", err)
				}
			case common.TYPE_GOPLUGIN:
				if col, err := parseGoPlugin(b); err == nil {
					col.Namespace = ns
					gopluginCollectors[ns+"|"+col.Name] = col
				} else {
					log.Warning("parse goplugin collection failed: ", err)
				}
			default:
				if interval, ok := item["interval"]; ok {
					var intervalInt int
					if intervalInt, err = strconv.Atoi(interval); err == nil {
						intervals[monitorType] = intervalInt
					} else {
						log.Warning("convert interval to int failed: ", err)
					}
				}
			}
		}
		//getAlarmPlugin(ns, pluginCollectors, pluginInfo)
	}

	common.SetPluginInfo(pluginInfo)
	for _, t := range append(common.SYS_TYPES, common.TYPE_PORT, common.TYPE_PROC) {
		if intervals[t] == 0 {
			intervals[t] = common.DEFAULT_INTERVAL[t]
		}
	}
	return
}

// func getAlarmPlugin(ns string, pluginCollectors map[string]plugins.Collector, pluginInfo map[string]bool) {
// 	url := fmt.Sprintf("http://%s/api/v1/resource?ns=%s&resource=alarm", common.Conf.RegistryAddr, ns)
// 	b, err := Get(url)
// 	if err != nil {
// 		return
// 	}
// 	var response models.Response
// 	err = json.Unmarshal(b, &response)
// 	if err != nil {
// 		log.Error("Unmarshal from alarm failed: ", err)
// 		return
// 	}

// 	alarms, ok := response.Data.([]map[string]interface{})
// 	if !ok {
// 		err = fmt.Errorf("response data is not a map slice type")
// 		return
// 	}

// 	for _, alarm := range alarms {
// 		ac, ok := alarm["actions"]
// 		if !ok {
// 			continue
// 		}
// 		actions, ok := ac.([]interface{})
// 		if !ok {
// 			continue
// 		}
// 		for _, a := range actions {
// 			action, ok := a.(map[string]interface{})
// 			if !ok {
// 				continue
// 			}
// 			t, ok := action["type"]
// 			if !ok {
// 				continue
// 			}
// 			if t1, ok := t.(string); ok && t1 == "AGENT" {
// 				b, err := json.Marshal(action)
// 				if err != nil {
// 					log.Error("json.Marshal action failed: ", err)
// 					continue
// 				}
// 				var col plugins.Collector
// 				err = json.Unmarshal(b, &col)
// 				if err != nil {
// 					log.Error("json.Unmarshal to plugins.Collector from alarm failed: ", err)
// 					continue
// 				}
// 				col, err = formatPlugin(col)
// 				if err != nil {
// 					continue
// 				}
// 				col.Cycle = 0
// 				pluginInfo[ns+"|"+col.Repo] = true
// 				//col.Namespace = ns
// 				//pluginCollectors[ns+"|"+col.ProjectName] = col
// 			}
// 		}
// 	}
// }

func parsePlugin(b []byte) (res plugins.Collector, err error) {
	if err = json.Unmarshal(b, &res); err != nil {
		return
	}
	var cycle int
	if cycle, err = strconv.Atoi(res.StrCycle); err != nil {
		return
	}
	res.Cycle = cycle
	res, err = formatPlugin(res)
	return
}

func formatPlugin(p plugins.Collector) (plugins.Collector, error) {
	if strings.Contains(p.Repo, ":") {
		s := strings.Split(p.Repo, ":")[1]
		p.Repo = s[:len(s)-4]
	}
	if strings.Count(p.Repo, "/") == 1 {
		p.ProjectName = strings.Split(p.Repo, "/")[1]
	}
	if p.Parameters != "" {
		for _, s := range strings.Split(p.Parameters, " ") {
			if s != "" {
				if strings.ContainsAny(s, ";|&<>`") {
					return p, errors.New("Invalid parameter")
				}
				p.Param = append(p.Param, s)
			}
		}
	}
	return p, nil
}

func parseProc(b []byte) (res common.ProcMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parsePort(b []byte) (res common.PortMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parseGoPlugin(b []byte) (res goplugin.Collector, err error) {
	err = json.Unmarshal(b, &res)
	return
}
<commit_msg>remove & in security check<commit_after>package loda

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/lodastack/agent/agent/common"
	"github.com/lodastack/agent/agent/goplugin"
	"github.com/lodastack/agent/agent/plugins"
	"github.com/lodastack/log"
)

var Zerotimes = 0

func Get(url string) (b []byte, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("Get url failed: %s  not found", url)
		return
	}
	b, err = ioutil.ReadAll(resp.Body)
	return
}

func Post(url string, data []byte) ([]byte, error) {
	body := bytes.NewBuffer([]byte(data))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json;charset=utf-8")

	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = fmt.Errorf("Post url failed: %s code: %d", url, res.StatusCode)
		return nil, err
	}

	result, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func Ns() ([]string, error) {
	var res []string
	host, err := common.Hostname()
	if err != nil {
		return res, err
	}
	//check hostname chaged
	changed, _ := common.HostnameChanged()
	if changed {
		return res, fmt.Errorf("hostname changed, skip fetch ns")
	}

	url := fmt.Sprintf("%s/api/v1/agent/ns", common.Conf.RegistryAddr)
	data := make(map[string]string)
	data["hostname"] = host
	data["ip"] = strings.Join(common.GetIpList(), ",")
	byteData, err := json.Marshal(data)
	b, err := Post(url, byteData)
	if err != nil {
		return res, err
	}

	type ResponseNS struct {
		Code int               `json:"httpstatus"`
		Data map[string]string `json:"data"`
	}
	var response ResponseNS
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Warning("json.Marshal Ns failed: ", err)
		return res, err
	}

	resp := response.Data

	var ids []string
	for ns, id := range resp {
		res = append(res, ns)
		ids = append(ids, id)
	}
	common.SetUUID(ids)
	return res, nil
}

func pullResources(ns string) (res []map[string]string, err error) {
	url := fmt.Sprintf("%s/api/v1/agent/resource?ns=%s&type=collect", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}

	type ResponseRes struct {
		Code int                 `json:"httpstatus"`
		Data []map[string]string `json:"data"`
	}
	var response ResponseRes
	err = json.Unmarshal(b, &response)
	if err != nil {
		return
	}
	res = response.Data

	if len(res) == 0 {
		err = fmt.Errorf("no items under this namespace")
		Zerotimes++
	} else {
		Zerotimes = 0
	}
	return
}

func MonitorItems() (ports []common.PortMonitor,
	procs []common.ProcMonitor,
	pluginCollectors map[string]plugins.Collector,
	gopluginCollectors map[string]goplugin.Collector,
	intervals map[string]int, err error) {
	nss := common.GetNamespaces()
	pluginCollectors = make(map[string]plugins.Collector)
	pluginInfo := make(map[string]bool)
	gopluginCollectors = make(map[string]goplugin.Collector)
	intervals = make(map[string]int)

	for _, ns := range nss {
		err = nil
		var items []map[string]string
		items, err = pullResources(ns)
		if err != nil {
			log.Error("failed to get resources from registry, ns: ", ns, " err: ", err)
			continue
		}

		for _, item := range items {
			monitorType, ok := item["measurement_type"]
			if !ok {
				log.Warning("measurement_type is not exist: ", item["measurement_type"])
				continue
			}
			b, err := json.Marshal(item)
			if err != nil {
				log.Warning("json.Marshal item failed: ", err)
				continue
			}
			switch monitorType {
			case common.TYPE_PORT:
				if port, err := parsePort(b); err == nil {
					port.Namespace = ns
					ports = append(ports, port)
				} else {
					log.Warning("parsePort failed: ", err)
				}
			case common.TYPE_PROC:
				if proc, err := parseProc(b); err == nil {
					proc.Namespace = ns
					procs = append(procs, proc)
				} else {
					log.Warning("parseProc failed: ", err)
				}
			case common.TYPE_PLUGIN:
				if col, err := parsePlugin(b); err == nil {
					col.Namespace = ns
					pluginCollectors[ns+"|"+col.ProjectName] = col
					pluginInfo[ns+"|"+col.Repo] = true
				} else {
					log.Warning("get plugin collection failed: ", err)
				}
			case common.TYPE_GOPLUGIN:
				if col, err := parseGoPlugin(b); err == nil {
					col.Namespace = ns
					gopluginCollectors[ns+"|"+col.Name] = col
				} else {
					log.Warning("parse goplugin collection failed: ", err)
				}
			default:
				if interval, ok := item["interval"]; ok {
					var intervalInt int
					if intervalInt, err = strconv.Atoi(interval); err == nil {
						intervals[monitorType] = intervalInt
					} else {
						log.Warning("convert interval to int failed: ", err)
					}
				}
			}
		}
		//getAlarmPlugin(ns, pluginCollectors, pluginInfo)
	}

	common.SetPluginInfo(pluginInfo)
	for _, t := range append(common.SYS_TYPES, common.TYPE_PORT, common.TYPE_PROC) {
		if intervals[t] == 0 {
			intervals[t] = common.DEFAULT_INTERVAL[t]
		}
	}
	return
}

// func getAlarmPlugin(ns string, pluginCollectors map[string]plugins.Collector, pluginInfo map[string]bool) {
// 	url := fmt.Sprintf("http://%s/api/v1/resource?ns=%s&resource=alarm", common.Conf.RegistryAddr, ns)
// 	b, err := Get(url)
// 	if err != nil {
// 		return
// 	}
// 	var response models.Response
// 	err = json.Unmarshal(b, &response)
// 	if err != nil {
// 		log.Error("Unmarshal from alarm failed: ", err)
// 		return
// 	}

// 	alarms, ok := response.Data.([]map[string]interface{})
// 	if !ok {
// 		err = fmt.Errorf("response data is not a map slice type")
// 		return
// 	}

// 	for _, alarm := range alarms {
// 		ac, ok := alarm["actions"]
// 		if !ok {
// 			continue
// 		}
// 		actions, ok := ac.([]interface{})
// 		if !ok {
// 			continue
// 		}
// 		for _, a := range actions {
// 			action, ok := a.(map[string]interface{})
// 			if !ok {
// 				continue
// 			}
// 			t, ok := action["type"]
// 			if !ok {
// 				continue
// 			}
// 			if t1, ok := t.(string); ok && t1 == "AGENT" {
// 				b, err := json.Marshal(action)
// 				if err != nil {
// 					log.Error("json.Marshal action failed: ", err)
// 					continue
// 				}
// 				var col plugins.Collector
// 				err = json.Unmarshal(b, &col)
// 				if err != nil {
// 					log.Error("json.Unmarshal to plugins.Collector from alarm failed: ", err)
// 					continue
// 				}
// 				col, err = formatPlugin(col)
// 				if err != nil {
// 					continue
// 				}
// 				col.Cycle = 0
// 				pluginInfo[ns+"|"+col.Repo] = true
// 				//col.Namespace = ns
// 				//pluginCollectors[ns+"|"+col.ProjectName] = col
// 			}
// 		}
// 	}
// }

func parsePlugin(b []byte) (res plugins.Collector, err error) {
	if err = json.Unmarshal(b, &res); err != nil {
		return
	}
	var cycle int
	if cycle, err = strconv.Atoi(res.StrCycle); err != nil {
		return
	}
	res.Cycle = cycle
	res, err = formatPlugin(res)
	return
}

func formatPlugin(p plugins.Collector) (plugins.Collector, error) {
	if strings.Contains(p.Repo, ":") {
		s := strings.Split(p.Repo, ":")[1]
		p.Repo = s[:len(s)-4]
	}
	if strings.Count(p.Repo, "/") == 1 {
		p.ProjectName = strings.Split(p.Repo, "/")[1]
	}
	if p.Parameters != "" {
		for _, s := range strings.Split(p.Parameters, " ") {
			if s != "" {
				if strings.ContainsAny(s, ";|<>`") {
					return p, errors.New("Invalid parameter")
				}
				p.Param = append(p.Param, s)
			}
		}
	}
	return p, nil
}

func parseProc(b []byte) (res common.ProcMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parsePort(b []byte) (res common.PortMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parseGoPlugin(b []byte) (res goplugin.Collector, err error) {
	err = json.Unmarshal(b, &res)
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/pilosa/pilosa/logger"
	"github.com/pilosa/pilosa/pql"
	"github.com/pilosa/pilosa/roaring"
	"github.com/pilosa/pilosa/stats"
	"github.com/pkg/errors"
)

// View layout modes.
const (
	viewStandard = "standard"

	viewBSIGroupPrefix = "bsig_"
)

// view represents a container for field data.
type view struct {
	mu    sync.RWMutex
	path  string
	index string
	field string
	name  string

	fieldType string
	cacheType string
	cacheSize uint32

	// Fragments by shard.
	fragments map[uint64]*fragment

	broadcaster  broadcaster
	stats        stats.StatsClient
	rowAttrStore AttrStore
	logger       logger.Logger
}

// newView returns a new instance of View.
func newView(path, index, field, name string, fieldOptions FieldOptions) *view {
	return &view{
		path:  path,
		index: index,
		field: field,
		name:  name,

		fieldType: fieldOptions.Type,
		cacheType: fieldOptions.CacheType,
		cacheSize: fieldOptions.CacheSize,

		fragments: make(map[uint64]*fragment),

		broadcaster: NopBroadcaster,
		stats:       stats.NopStatsClient,
		logger:      logger.NopLogger,
	}
}

// open opens and initializes the view.
func (v *view) open() error {

	// Never keep a cache for field views.
	if strings.HasPrefix(v.name, viewBSIGroupPrefix) {
		v.cacheType = CacheTypeNone
	}

	if err := func() error {
		// Ensure the view's path exists.
		if err := os.MkdirAll(v.path, 0777); err != nil {
			return errors.Wrap(err, "creating view directory")
		} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
			return errors.Wrap(err, "creating fragments directory")
		}

		if err := v.openFragments(); err != nil {
			return errors.Wrap(err, "opening fragments")
		}

		return nil
	}(); err != nil {
		v.close()
		return err
	}

	return nil
}

// openFragments opens and initializes the fragments inside the view.
func (v *view) openFragments() error {
	file, err := os.Open(filepath.Join(v.path, "fragments"))
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return errors.Wrap(err, "opening fragments directory")
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return errors.Wrap(err, "reading fragments directory")
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := v.newFragment(v.fragmentPath(shard), shard)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
		}
		frag.RowAttrStore = v.rowAttrStore
		v.fragments[frag.shard] = frag
	}

	return nil
}

// close closes the view and its fragments.
func (v *view) close() error {
	v.mu.Lock()
	defer v.mu.Unlock()

	// Close all fragments.
	for _, frag := range v.fragments {
		if err := frag.Close(); err != nil {
			return errors.Wrap(err, "closing fragment")
		}
	}
	v.fragments = make(map[uint64]*fragment)

	return nil
}

// availableShards returns a bitmap of shards which contain data.
func (v *view) availableShards() *roaring.Bitmap {
	v.mu.RLock()
	defer v.mu.RUnlock()

	b := roaring.NewBitmap()
	for shard := range v.fragments {
		b.Add(shard) // ignore error, no writer attached
	}
	return b
}

// fragmentPath returns the path to a fragment in the view.
func (v *view) fragmentPath(shard uint64) string {
	return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10))
}

// Fragment returns a fragment in the view by shard.
func (v *view) Fragment(shard uint64) *fragment {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.fragment(shard)
}

func (v *view) fragment(shard uint64) *fragment { return v.fragments[shard] }

// allFragments returns a list of all fragments in the view.
func (v *view) allFragments() []*fragment {
	v.mu.Lock()
	defer v.mu.Unlock()

	other := make([]*fragment, 0, len(v.fragments))
	for _, fragment := range v.fragments {
		other = append(other, fragment)
	}
	return other
}

// recalculateCaches recalculates the cache on every fragment in the view.
func (v *view) recalculateCaches() {
	for _, fragment := range v.allFragments() {
		fragment.RecalculateCache()
	}
}

// CreateFragmentIfNotExists returns a fragment in the view by shard.
func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
	frag, msg, err := v.createFragmentIfNotExists(shard)

	if err == nil && msg != nil {
		// Broadcast a message that a new max shard was just created.
		if err = v.broadcaster.SendSync(msg); err != nil {
			v.mu.Lock()
			delete(v.fragments, shard)
			v.mu.Unlock()
			frag.close()
			return nil, errors.Wrap(err, "sending createshard message")
		}
	}

	return frag, err
}

func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardMessage, error) {
	v.mu.Lock()
	defer v.mu.Unlock()
	// Find fragment in cache first.
	if frag := v.fragments[shard]; frag != nil {
		return frag, nil, nil
	}

	// Initialize and open fragment.
	frag := v.newFragment(v.fragmentPath(shard), shard)
	if err := frag.Open(); err != nil {
		return nil, nil, errors.Wrap(err, "opening fragment")
	}
	frag.RowAttrStore = v.rowAttrStore

	msg := &CreateShardMessage{
		Index: v.index,
		Field: v.field,
		Shard: shard,
	}
	v.fragments[shard] = frag

	// Save to lookup.
	return frag, msg, nil
}

func (v *view) newFragment(path string, shard uint64) *fragment {
	frag := newFragment(path, v.index, v.field, v.name, shard)
	frag.CacheType = v.cacheType
	frag.CacheSize = v.cacheSize
	frag.Logger = v.logger
	frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
	if v.fieldType == FieldTypeMutex {
		frag.mutexVector = newRowsVector(frag)
	} else if v.fieldType == FieldTypeBool {
		frag.mutexVector = newBoolVector(frag)
	}
	return frag
}

// deleteFragment removes the fragment from the view.
func (v *view) deleteFragment(shard uint64) error {

	fragment := v.fragments[shard]
	if fragment == nil {
		return ErrFragmentNotFound
	}

	v.logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)

	// Close data files before deletion.
	if err := fragment.Close(); err != nil {
		return errors.Wrap(err, "closing fragment")
	}

	// Delete fragment file.
	if err := os.Remove(fragment.path); err != nil {
		return errors.Wrap(err, "deleting fragment file")
	}

	// Delete fragment cache file.
	if err := os.Remove(fragment.cachePath()); err != nil {
		v.logger.Printf("no cache file to delete for shard %d", shard)
	}

	delete(v.fragments, shard)

	return nil
}

// row returns a row for a shard of the view.
func (v *view) row(rowID uint64) *Row {
	row := NewRow()
	for _, frag := range v.allFragments() {
		fr := frag.row(rowID)
		if fr == nil {
			continue
		}
		row.Merge(fr)
	}
	return row

}

// setBit sets a bit within the view.
func (v *view) setBit(rowID, columnID uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return changed, err
	}
	return frag.setBit(rowID, columnID)
}

// clearBit clears a bit within the view.
func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, found := v.fragments[shard]
	if !found {
		return false, nil
	}
	return frag.clearBit(rowID, columnID)
}

// value uses a column of bits to read a multi-bit value.
func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return value, exists, err
	}
	return frag.value(columnID, bitDepth)
}

// setValue uses a column of bits to set a multi-bit value.
func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return changed, err
	}
	return frag.setValue(columnID, bitDepth, value)
}

// sum returns the sum & count of a field.
func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
	for _, f := range v.allFragments() {
		fsum, fcount, err := f.sum(filter, bitDepth)
		if err != nil {
			return sum, count, err
		}
		sum += fsum
		count += fcount
	}
	return sum, count, nil
}

// min returns the min and count of a field.
func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
	var minHasValue bool
	for _, f := range v.allFragments() {
		fmin, fcount, err := f.min(filter, bitDepth)
		if err != nil {
			return min, count, err
		}
		// Don't consider a min based on zero columns.
		if fcount == 0 {
			continue
		}

		if !minHasValue {
			min = fmin
			minHasValue = true
			count += fcount
			continue
		}

		if fmin < min {
			min = fmin
			count += fcount
		}
	}
	return min, count, nil
}

// max returns the max and count of a field.
func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
	for _, f := range v.allFragments() {
		fmax, fcount, err := f.max(filter, bitDepth)
		if err != nil {
			return max, count, err
		}
		if fcount > 0 && fmax > max {
			max = fmax
			count += fcount
		}
	}
	return max, count, nil
}

// rangeOp returns rows with a field value encoding matching the predicate.
func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
	r := NewRow()
	for _, frag := range v.allFragments() {
		other, err := frag.rangeOp(op, bitDepth, predicate)
		if err != nil {
			return nil, err
		}
		r = r.Union(other)
	}
	return r, nil
}

// ViewInfo represents schema information for a view.
type ViewInfo struct {
	Name string `json:"name"`
}

type viewInfoSlice []*ViewInfo

func (p viewInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int           { return len(p) }
func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<commit_msg>improve comments<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/pilosa/pilosa/logger"
	"github.com/pilosa/pilosa/pql"
	"github.com/pilosa/pilosa/roaring"
	"github.com/pilosa/pilosa/stats"
	"github.com/pkg/errors"
)

// View layout modes.
const (
	viewStandard = "standard"

	viewBSIGroupPrefix = "bsig_"
)

// view represents a container for field data.
type view struct {
	mu    sync.RWMutex
	path  string
	index string
	field string
	name  string

	fieldType string
	cacheType string
	cacheSize uint32

	// Fragments by shard.
	fragments map[uint64]*fragment

	broadcaster  broadcaster
	stats        stats.StatsClient
	rowAttrStore AttrStore
	logger       logger.Logger
}

// newView returns a new instance of View.
func newView(path, index, field, name string, fieldOptions FieldOptions) *view {
	return &view{
		path:  path,
		index: index,
		field: field,
		name:  name,

		fieldType: fieldOptions.Type,
		cacheType: fieldOptions.CacheType,
		cacheSize: fieldOptions.CacheSize,

		fragments: make(map[uint64]*fragment),

		broadcaster: NopBroadcaster,
		stats:       stats.NopStatsClient,
		logger:      logger.NopLogger,
	}
}

// open opens and initializes the view.
func (v *view) open() error {

	// Never keep a cache for field views.
	if strings.HasPrefix(v.name, viewBSIGroupPrefix) {
		v.cacheType = CacheTypeNone
	}

	if err := func() error {
		// Ensure the view's path exists.
		if err := os.MkdirAll(v.path, 0777); err != nil {
			return errors.Wrap(err, "creating view directory")
		} else if err := os.MkdirAll(filepath.Join(v.path, "fragments"), 0777); err != nil {
			return errors.Wrap(err, "creating fragments directory")
		}

		if err := v.openFragments(); err != nil {
			return errors.Wrap(err, "opening fragments")
		}

		return nil
	}(); err != nil {
		v.close()
		return err
	}

	return nil
}

// openFragments opens and initializes the fragments inside the view.
func (v *view) openFragments() error {
	file, err := os.Open(filepath.Join(v.path, "fragments"))
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return errors.Wrap(err, "opening fragments directory")
	}
	defer file.Close()

	fis, err := file.Readdir(0)
	if err != nil {
		return errors.Wrap(err, "reading fragments directory")
	}

	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}

		// Parse filename into integer.
		shard, err := strconv.ParseUint(filepath.Base(fi.Name()), 10, 64)
		if err != nil {
			continue
		}

		frag := v.newFragment(v.fragmentPath(shard), shard)
		if err := frag.Open(); err != nil {
			return fmt.Errorf("open fragment: shard=%d, err=%s", frag.shard, err)
		}
		frag.RowAttrStore = v.rowAttrStore
		v.fragments[frag.shard] = frag
	}

	return nil
}

// close closes the view and its fragments.
func (v *view) close() error {
	v.mu.Lock()
	defer v.mu.Unlock()

	// Close all fragments.
	for _, frag := range v.fragments {
		if err := frag.Close(); err != nil {
			return errors.Wrap(err, "closing fragment")
		}
	}
	v.fragments = make(map[uint64]*fragment)

	return nil
}

// availableShards returns a bitmap of shards which contain data.
func (v *view) availableShards() *roaring.Bitmap {
	v.mu.RLock()
	defer v.mu.RUnlock()

	b := roaring.NewBitmap()
	for shard := range v.fragments {
		b.Add(shard) // ignore error, no writer attached
	}
	return b
}

// fragmentPath returns the path to a fragment in the view.
func (v *view) fragmentPath(shard uint64) string {
	return filepath.Join(v.path, "fragments", strconv.FormatUint(shard, 10))
}

// Fragment returns a fragment in the view by shard.
func (v *view) Fragment(shard uint64) *fragment {
	v.mu.RLock()
	defer v.mu.RUnlock()
	return v.fragment(shard)
}

func (v *view) fragment(shard uint64) *fragment { return v.fragments[shard] }

// allFragments returns a list of all fragments in the view.
func (v *view) allFragments() []*fragment {
	v.mu.Lock()
	defer v.mu.Unlock()

	other := make([]*fragment, 0, len(v.fragments))
	for _, fragment := range v.fragments {
		other = append(other, fragment)
	}
	return other
}

// recalculateCaches recalculates the cache on every fragment in the view.
func (v *view) recalculateCaches() {
	for _, fragment := range v.allFragments() {
		fragment.RecalculateCache()
	}
}

// CreateFragmentIfNotExists returns a fragment in the view by shard.
func (v *view) CreateFragmentIfNotExists(shard uint64) (*fragment, error) {
	frag, msg, err := v.createFragmentIfNotExists(shard)

	// if msg is not nil, then a new shard was created
	if err == nil && msg != nil {
		// Broadcast a message that a new max shard was just created.
		if err = v.broadcaster.SendSync(msg); err != nil {
			v.mu.Lock()
			delete(v.fragments, shard)
			v.mu.Unlock()
			frag.close()
			return nil, errors.Wrap(err, "sending createshard message")
		}
	}

	return frag, err
}

func (v *view) createFragmentIfNotExists(shard uint64) (*fragment, *CreateShardMessage, error) {
	v.mu.Lock()
	defer v.mu.Unlock()
	// Find fragment in cache first.
	if frag := v.fragments[shard]; frag != nil {
		return frag, nil, nil
	}

	// Initialize and open fragment.
	frag := v.newFragment(v.fragmentPath(shard), shard)
	if err := frag.Open(); err != nil {
		return nil, nil, errors.Wrap(err, "opening fragment")
	}
	frag.RowAttrStore = v.rowAttrStore

	msg := &CreateShardMessage{
		Index: v.index,
		Field: v.field,
		Shard: shard,
	}

	// Save to lookup.
	v.fragments[shard] = frag

	return frag, msg, nil
}

func (v *view) newFragment(path string, shard uint64) *fragment {
	frag := newFragment(path, v.index, v.field, v.name, shard)
	frag.CacheType = v.cacheType
	frag.CacheSize = v.cacheSize
	frag.Logger = v.logger
	frag.stats = v.stats.WithTags(fmt.Sprintf("shard:%d", shard))
	if v.fieldType == FieldTypeMutex {
		frag.mutexVector = newRowsVector(frag)
	} else if v.fieldType == FieldTypeBool {
		frag.mutexVector = newBoolVector(frag)
	}
	return frag
}

// deleteFragment removes the fragment from the view.
func (v *view) deleteFragment(shard uint64) error {

	fragment := v.fragments[shard]
	if fragment == nil {
		return ErrFragmentNotFound
	}

	v.logger.Printf("delete fragment: (%s/%s/%s) %d", v.index, v.field, v.name, shard)

	// Close data files before deletion.
	if err := fragment.Close(); err != nil {
		return errors.Wrap(err, "closing fragment")
	}

	// Delete fragment file.
	if err := os.Remove(fragment.path); err != nil {
		return errors.Wrap(err, "deleting fragment file")
	}

	// Delete fragment cache file.
	if err := os.Remove(fragment.cachePath()); err != nil {
		v.logger.Printf("no cache file to delete for shard %d", shard)
	}

	delete(v.fragments, shard)

	return nil
}

// row returns a row for a shard of the view.
func (v *view) row(rowID uint64) *Row {
	row := NewRow()
	for _, frag := range v.allFragments() {
		fr := frag.row(rowID)
		if fr == nil {
			continue
		}
		row.Merge(fr)
	}
	return row

}

// setBit sets a bit within the view.
func (v *view) setBit(rowID, columnID uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return changed, err
	}
	return frag.setBit(rowID, columnID)
}

// clearBit clears a bit within the view.
func (v *view) clearBit(rowID, columnID uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, found := v.fragments[shard]
	if !found {
		return false, nil
	}
	return frag.clearBit(rowID, columnID)
}

// value uses a column of bits to read a multi-bit value.
func (v *view) value(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return value, exists, err
	}
	return frag.value(columnID, bitDepth)
}

// setValue uses a column of bits to set a multi-bit value.
func (v *view) setValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
	shard := columnID / ShardWidth
	frag, err := v.CreateFragmentIfNotExists(shard)
	if err != nil {
		return changed, err
	}
	return frag.setValue(columnID, bitDepth, value)
}

// sum returns the sum & count of a field.
func (v *view) sum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
	for _, f := range v.allFragments() {
		fsum, fcount, err := f.sum(filter, bitDepth)
		if err != nil {
			return sum, count, err
		}
		sum += fsum
		count += fcount
	}
	return sum, count, nil
}

// min returns the min and count of a field.
func (v *view) min(filter *Row, bitDepth uint) (min, count uint64, err error) {
	var minHasValue bool
	for _, f := range v.allFragments() {
		fmin, fcount, err := f.min(filter, bitDepth)
		if err != nil {
			return min, count, err
		}
		// Don't consider a min based on zero columns.
		if fcount == 0 {
			continue
		}

		if !minHasValue {
			min = fmin
			minHasValue = true
			count += fcount
			continue
		}

		if fmin < min {
			min = fmin
			count += fcount
		}
	}
	return min, count, nil
}

// max returns the max and count of a field.
func (v *view) max(filter *Row, bitDepth uint) (max, count uint64, err error) {
	for _, f := range v.allFragments() {
		fmax, fcount, err := f.max(filter, bitDepth)
		if err != nil {
			return max, count, err
		}
		if fcount > 0 && fmax > max {
			max = fmax
			count += fcount
		}
	}
	return max, count, nil
}

// rangeOp returns rows with a field value encoding matching the predicate.
func (v *view) rangeOp(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
	r := NewRow()
	for _, frag := range v.allFragments() {
		other, err := frag.rangeOp(op, bitDepth, predicate)
		if err != nil {
			return nil, err
		}
		r = r.Union(other)
	}
	return r, nil
}

// ViewInfo represents schema information for a view.
type ViewInfo struct {
	Name string `json:"name"`
}

type viewInfoSlice []*ViewInfo

func (p viewInfoSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p viewInfoSlice) Len() int           { return len(p) }
func (p viewInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
<|endoftext|>


<commit_before>package jetpack

import "bytes"
import "encoding/json"
import "fmt"
import "io/ioutil"
import "os"
import "strconv"
import "strings"
import "text/template"
import "time"

import "github.com/appc/spec/schema"
import "github.com/appc/spec/schema/types"
import "github.com/juju/errors"

import "github.com/3ofcoins/jetpack/run"
import "github.com/3ofcoins/jetpack/zfs"

var jailConfTmpl *template.Template

func init() {
	tmpl, err := template.New("jail.conf").Parse(
		`"{{.JailName}}" {
  path = "{{.Dataset.Mountpoint}}/rootfs";
  devfs_ruleset="4";
  exec.clean="true";
  host.hostname="{{(.GetAnnotation "hostname" .Manifest.UUID.String)}}";
  host.hostuuid="{{.Manifest.UUID}}";
  interface="{{.Manager.Interface}}";
  ip4.addr="{{(.GetAnnotation "ip-address" "CAN'T HAPPEN")}}";
  mount.devfs="true";
  persist="true";
{{ range $param, $value := .JailParameters }}
  {{$param}} = "{{$value}}";
{{ end }}
}
`)
	if err != nil {
		panic(err)
	} else {
		jailConfTmpl = tmpl
	}
}

var ErrContainerIsEmpty = errors.New("Container is empty")

type ContainerStatus uint

const (
	ContainerStatusInvalid ContainerStatus = iota
	ContainerStatusRunning
	ContainerStatusDying
	ContainerStatusStopped
)

var containerStatusNames = []string{
	ContainerStatusInvalid: "invalid",
	ContainerStatusRunning: "running",
	ContainerStatusDying:   "dying",
	ContainerStatusStopped: "stopped",
}

func (cs ContainerStatus) String() string {
	if int(cs) < len(containerStatusNames) {
		return containerStatusNames[cs]
	}
	return fmt.Sprintf("ContainerStatus[%d]", cs)
}

type Container struct {
	Dataset  *zfs.Dataset                    `json:"-"`
	Manifest schema.ContainerRuntimeManifest `json:"-"`
	Manager  *ContainerManager               `json:"-"`

	JailParameters map[string]string

	image *Image
}

func NewContainer(ds *zfs.Dataset, mgr *ContainerManager) *Container {
	return &Container{Dataset: ds, Manager: mgr, JailParameters: make(map[string]string)}
}

func GetContainer(ds *zfs.Dataset, mgr *ContainerManager) (*Container, error) {
	c := NewContainer(ds, mgr)
	if err := c.Load(); err != nil {
		return nil, err
	} else {
		return c, nil
	}
}

func (c *Container) IsEmpty() bool {
	_, err := os.Stat(c.Dataset.Path("manifest"))
	return os.IsNotExist(err)
}

func (c *Container) IsLoaded() bool {
	return !c.Manifest.ACVersion.Empty()
}

func (c *Container) Load() error {
	if c.IsLoaded() {
		return errors.New("Already loaded")
	}

	if c.IsEmpty() {
		return ErrContainerIsEmpty
	}

	if err := c.readManifest(); err != nil {
		return errors.Trace(err)
	}

	if len(c.Manifest.Apps) == 0 {
		return errors.Errorf("No application set?")
	}

	if len(c.Manifest.Apps) > 1 {
		return errors.Errorf("TODO: Multi-application containers are not supported")
	}

	if len(c.Manifest.Isolators) != 0 || len(c.Manifest.Apps[0].Isolators) != 0 {
		return errors.Errorf("TODO: isolators are not supported")
	}
	return nil
}

func (c *Container) readManifest() error {
	manifestJSON, err := ioutil.ReadFile(c.Dataset.Path("manifest"))
	if err != nil {
		return errors.Trace(err)
	}

	if err = json.Unmarshal(manifestJSON, &c.Manifest); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (c *Container) Save() error {
	manifestJSON, err := json.Marshal(c.Manifest)
	if err != nil {
		return errors.Trace(err)
	}
	return errors.Trace(ioutil.WriteFile(c.Dataset.Path("manifest"), manifestJSON, 0400))
}

func (c *Container) findVolume(name types.ACName) *types.Volume {
	for _, vol := range c.Manifest.Volumes {
		for _, fulfills := range vol.Fulfills {
			if fulfills == name {
				return &vol
			}
		}
	}
	return nil
}

func (c *Container) Prep() error {
	img, err := c.GetImage()
	if err != nil {
		return errors.Trace(err)
	}

	if app := img.Manifest.App; app != nil && len(app.MountPoints) > 0 {
		fstab := make([]string, len(app.MountPoints))
		for i, mnt := range app.MountPoints {
			if vol := c.findVolume(mnt.Name); vol == nil {
				return errors.Errorf("No volume found for %v", mnt.Name)
			} else {
				opts := "rw"
				if vol.ReadOnly {
					opts = "ro"
				}
				fstab[i] = fmt.Sprintf("%v %v nullfs %v 0 0\n",
					vol.Source,
					c.Dataset.Path("rootfs", mnt.Path),
					opts,
				)
			}
		}
		fstabPath := c.Dataset.Path("fstab")
		if err := ioutil.WriteFile(fstabPath, []byte(strings.Join(fstab, "")), 0600); err != nil {
			return errors.Trace(err)
		}
		c.JailParameters["mount.fstab"] = fstabPath
	}

	if bb, err := ioutil.ReadFile("/etc/resolv.conf"); err != nil {
		return errors.Trace(err)
	} else {
		if err := ioutil.WriteFile(c.Dataset.Path("rootfs/etc/resolv.conf"), bb, 0644); err != nil {
			return errors.Trace(err)
		}
	}

	jc, err := os.OpenFile(c.Dataset.Path("jail.conf"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0400)
	if err != nil {
		return errors.Trace(err)
	}
	defer jc.Close()

	return errors.Trace(jailConfTmpl.Execute(jc, c))
}

func (c *Container) GetAnnotation(key, defval string) string {
	if val, ok := c.Manifest.Annotations[types.ACName(key)]; ok {
		return val
	} else {
		return defval
	}
}

func (c *Container) Status() ContainerStatus {
	if status, err := c.GetJailStatus(false); err != nil {
		panic(err)
	} else {
		if status == NoJailStatus {
			return ContainerStatusStopped
		}
		if status.Dying {
			return ContainerStatusDying
		}
		return ContainerStatusRunning
	}
}

func (c *Container) Kill() error {
	t0 := time.Now()
retry:
	switch status := c.Status(); status {
	case ContainerStatusStopped:
		// All's fine
		return nil
	case ContainerStatusRunning:
		if err := c.RunJail("-r"); err != nil {
			return errors.Trace(err)
		}
		goto retry
	case ContainerStatusDying:
		// TODO: UI? Log?
		fmt.Printf("Container dying since %v, waiting...\n", time.Now().Sub(t0))
		time.Sleep(2500 * time.Millisecond)
		goto retry
	default:
		return errors.Errorf("Container is %v, I am confused", status)
	}
}

func (c *Container) Destroy() error {
	return c.Dataset.Destroy("-r")
}

func (c *Container) JailName() string {
	return c.Manager.JailNamePrefix + c.Manifest.UUID.String()
}

func (c *Container) GetJailStatus(refresh bool) (JailStatus, error) {
	return c.Manager.Host.GetJailStatus(c.JailName(), refresh)
}

func (c *Container) Jid() int {
	if status, err := c.GetJailStatus(false); err != nil {
		panic(err) // do we need to?
	} else {
		return status.Jid
	}
}

func (c *Container) RunJail(op string) error {
	if err := c.Prep(); err != nil {
		return err
	}
	return run.Command("jail", "-f", c.Dataset.Path("jail.conf"), "-v", op, c.JailName()).Run()
}

func (c *Container) GetImage() (*Image, error) {
	if c.image == nil {
		hash := c.Manifest.Apps[0].ImageID.Val
		if !strings.HasPrefix(hash, "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") {
			return nil, errors.New("FIXME: sha512 is a real checksum, not wrapped UUID, and I am confused now.")
		}
		hash = hash[128-32:]
		uuid := strings.Join([]string{hash[:8], hash[8:12], hash[12:16], hash[16:20], hash[20:]}, "-")
		if img, err := c.Manager.Host.Images.Get(uuid); err != nil {
			return nil, errors.Trace(err)
		} else {
			c.image = img
		}
	}
	return c.image, nil
}

func (c *Container) Run(app *types.App) (err1 error) {
	if err := c.RunJail("-c"); err != nil {
		return errors.Trace(err)
	}
	defer func() {
		if err := c.Kill(); err != nil {
			if err1 != nil {
				err1 = errors.Wrap(err1, errors.Trace(err))
			} else {
				err1 = errors.Trace(err)
			}
		}
	}()
	return c.Stage2(app)
}

func (c *Container) Stage2(app *types.App) error {
	img, err := c.GetImage()
	if err != nil {
		return errors.Trace(err)
	}

	if app == nil {
		app = img.GetApp()
	}

	jid := c.Jid()
	if jid == 0 {
		return errors.New("Not started")
	}

	user := app.User
	if user == "" {
		user = "root"
	}

	args := []string{
		"-jid", strconv.Itoa(jid),
		"-user", user,
		"-group", app.Group,
		"-name", string(img.Manifest.Name),
	}

	for k, v := range app.Environment {
		args = append(args, "-setenv", k+"="+v)
	}

	args = append(args, app.Exec...)

	// FIXME:libexec
	return run.Command("/home/japhy/Go/src/github.com/3ofcoins/jetpack/bin/stage2", args...).Run()
}

type ContainerSlice []*Container

func (cc ContainerSlice) Len() int { return len(cc) }
func (cc ContainerSlice) Less(i, j int) bool {
	return bytes.Compare(cc[i].Manifest.UUID[:], cc[j].Manifest.UUID[:]) < 0
}
func (cc ContainerSlice) Swap(i, j int) { cc[i], cc[j] = cc[j], cc[i] }

func (cc ContainerSlice) Table() [][]string {
	rows := make([][]string, len(cc)+1)
	rows[0] = []string{"UUID", "IMAGE", "APP", "IP", "STATUS"}
	for i, c := range cc {
		imageID := ""
		if img, err := c.GetImage(); err != nil {
			imageID = fmt.Sprintf("[%v]", err)
		} else {
			imageID = img.UUID.String()
		}

		appName := ""
		if len(c.Manifest.Apps) > 0 {
			appName = string(c.Manifest.Apps[0].Name)
		}
		rows[i+1] = []string{
			c.Manifest.UUID.String(),
			imageID,
			appName,
			c.GetAnnotation("ip-address", ""),
			c.Status().String(),
		}
	}
	return rows
}
<commit_msg>ZFS-based image detection<commit_after>package jetpack

import "bytes"
import "encoding/json"
import "fmt"
import "io/ioutil"
import "os"
import "path"
import "strconv"
import "strings"
import "text/template"
import "time"

import "github.com/appc/spec/schema"
import "github.com/appc/spec/schema/types"
import "github.com/juju/errors"

import "github.com/3ofcoins/jetpack/run"
import "github.com/3ofcoins/jetpack/zfs"

var jailConfTmpl *template.Template

func init() {
	tmpl, err := template.New("jail.conf").Parse(
		`"{{.JailName}}" {
  path = "{{.Dataset.Mountpoint}}/rootfs";
  devfs_ruleset="4";
  exec.clean="true";
  host.hostname="{{(.GetAnnotation "hostname" .Manifest.UUID.String)}}";
  host.hostuuid="{{.Manifest.UUID}}";
  interface="{{.Manager.Interface}}";
  ip4.addr="{{(.GetAnnotation "ip-address" "CAN'T HAPPEN")}}";
  mount.devfs="true";
  persist="true";
{{ range $param, $value := .JailParameters }}
  {{$param}} = "{{$value}}";
{{ end }}
}
`)
	if err != nil {
		panic(err)
	} else {
		jailConfTmpl = tmpl
	}
}

var ErrContainerIsEmpty = errors.New("Container is empty")

type ContainerStatus uint

const (
	ContainerStatusInvalid ContainerStatus = iota
	ContainerStatusRunning
	ContainerStatusDying
	ContainerStatusStopped
)

var containerStatusNames = []string{
	ContainerStatusInvalid: "invalid",
	ContainerStatusRunning: "running",
	ContainerStatusDying:   "dying",
	ContainerStatusStopped: "stopped",
}

func (cs ContainerStatus) String() string {
	if int(cs) < len(containerStatusNames) {
		return containerStatusNames[cs]
	}
	return fmt.Sprintf("ContainerStatus[%d]", cs)
}

type Container struct {
	Dataset  *zfs.Dataset                    `json:"-"`
	Manifest schema.ContainerRuntimeManifest `json:"-"`
	Manager  *ContainerManager               `json:"-"`

	JailParameters map[string]string

	image *Image
}

func NewContainer(ds *zfs.Dataset, mgr *ContainerManager) *Container {
	return &Container{Dataset: ds, Manager: mgr, JailParameters: make(map[string]string)}
}

func GetContainer(ds *zfs.Dataset, mgr *ContainerManager) (*Container, error) {
	c := NewContainer(ds, mgr)
	if err := c.Load(); err != nil {
		return nil, err
	} else {
		return c, nil
	}
}

func (c *Container) IsEmpty() bool {
	_, err := os.Stat(c.Dataset.Path("manifest"))
	return os.IsNotExist(err)
}

func (c *Container) IsLoaded() bool {
	return !c.Manifest.ACVersion.Empty()
}

func (c *Container) Load() error {
	if c.IsLoaded() {
		return errors.New("Already loaded")
	}

	if c.IsEmpty() {
		return ErrContainerIsEmpty
	}

	if err := c.readManifest(); err != nil {
		return errors.Trace(err)
	}

	if len(c.Manifest.Apps) == 0 {
		return errors.Errorf("No application set?")
	}

	if len(c.Manifest.Apps) > 1 {
		return errors.Errorf("TODO: Multi-application containers are not supported")
	}

	if len(c.Manifest.Isolators) != 0 || len(c.Manifest.Apps[0].Isolators) != 0 {
		return errors.Errorf("TODO: isolators are not supported")
	}
	return nil
}

func (c *Container) readManifest() error {
	manifestJSON, err := ioutil.ReadFile(c.Dataset.Path("manifest"))
	if err != nil {
		return errors.Trace(err)
	}

	if err = json.Unmarshal(manifestJSON, &c.Manifest); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (c *Container) Save() error {
	manifestJSON, err := json.Marshal(c.Manifest)
	if err != nil {
		return errors.Trace(err)
	}
	return errors.Trace(ioutil.WriteFile(c.Dataset.Path("manifest"), manifestJSON, 0400))
}

func (c *Container) findVolume(name types.ACName) *types.Volume {
	for _, vol := range c.Manifest.Volumes {
		for _, fulfills := range vol.Fulfills {
			if fulfills == name {
				return &vol
			}
		}
	}
	return nil
}

func (c *Container) Prep() error {
	img, err := c.GetImage()
	if err != nil {
		return errors.Trace(err)
	}

	if app := img.Manifest.App; app != nil && len(app.MountPoints) > 0 {
		fstab := make([]string, len(app.MountPoints))
		for i, mnt := range app.MountPoints {
			if vol := c.findVolume(mnt.Name); vol == nil {
				return errors.Errorf("No volume found for %v", mnt.Name)
			} else {
				opts := "rw"
				if vol.ReadOnly {
					opts = "ro"
				}
				fstab[i] = fmt.Sprintf("%v %v nullfs %v 0 0\n",
					vol.Source,
					c.Dataset.Path("rootfs", mnt.Path),
					opts,
				)
			}
		}
		fstabPath := c.Dataset.Path("fstab")
		if err := ioutil.WriteFile(fstabPath, []byte(strings.Join(fstab, "")), 0600); err != nil {
			return errors.Trace(err)
		}
		c.JailParameters["mount.fstab"] = fstabPath
	}

	if bb, err := ioutil.ReadFile("/etc/resolv.conf"); err != nil {
		return errors.Trace(err)
	} else {
		if err := ioutil.WriteFile(c.Dataset.Path("rootfs/etc/resolv.conf"), bb, 0644); err != nil {
			return errors.Trace(err)
		}
	}

	jc, err := os.OpenFile(c.Dataset.Path("jail.conf"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0400)
	if err != nil {
		return errors.Trace(err)
	}
	defer jc.Close()

	return errors.Trace(jailConfTmpl.Execute(jc, c))
}

func (c *Container) GetAnnotation(key, defval string) string {
	if val, ok := c.Manifest.Annotations[types.ACName(key)]; ok {
		return val
	} else {
		return defval
	}
}

func (c *Container) Status() ContainerStatus {
	if status, err := c.GetJailStatus(false); err != nil {
		panic(err)
	} else {
		if status == NoJailStatus {
			return ContainerStatusStopped
		}
		if status.Dying {
			return ContainerStatusDying
		}
		return ContainerStatusRunning
	}
}

func (c *Container) runJail(op string) error {
	if err := c.Prep(); err != nil {
		return err
	}
	return run.Command("jail", "-f", c.Dataset.Path("jail.conf"), "-v", op, c.JailName()).Run()
}

func (c *Container) Spawn() error {
	return errors.Trace(c.runJail("-c"))
}

func (c *Container) Kill() error {
	t0 := time.Now()
retry:
	switch status := c.Status(); status {
	case ContainerStatusStopped:
		// All's fine
		return nil
	case ContainerStatusRunning:
		if err := c.runJail("-r"); err != nil {
			return errors.Trace(err)
		}
		goto retry
	case ContainerStatusDying:
		// TODO: UI? Log?
		fmt.Printf("Container dying since %v, waiting...\n", time.Now().Sub(t0))
		time.Sleep(2500 * time.Millisecond)
		goto retry
	default:
		return errors.Errorf("Container is %v, I am confused", status)
	}
}

func (c *Container) Destroy() error {
	return c.Dataset.Destroy("-r")
}

func (c *Container) JailName() string {
	return c.Manager.JailNamePrefix + c.Manifest.UUID.String()
}

func (c *Container) GetJailStatus(refresh bool) (JailStatus, error) {
	return c.Manager.Host.GetJailStatus(c.JailName(), refresh)
}

func (c *Container) Jid() int {
	if status, err := c.GetJailStatus(false); err != nil {
		panic(err) // do we need to?
	} else {
		return status.Jid
	}
}

func (c *Container) imageUUID() string {
	return strings.Split(path.Base(c.Dataset.Origin), "@")[0]
}

func (c *Container) GetImage() (*Image, error) {
	if c.image == nil {
		if img, err := c.Manager.Host.Images.Get(c.imageUUID()); err != nil {
			return nil, errors.Trace(err)
		} else {
			c.image = img
		}
	}
	return c.image, nil
}

func (c *Container) Run(app *types.App) (err1 error) {
	if err := c.Spawn(); err != nil {
		return errors.Trace(err)
	}
	defer func() {
		if err := c.Kill(); err != nil {
			if err1 != nil {
				err1 = errors.Wrap(err1, errors.Trace(err))
			} else {
				err1 = errors.Trace(err)
			}
		}
	}()
	return c.Stage2(app)
}

func (c *Container) Stage2(app *types.App) error {
	img, err := c.GetImage()
	if err != nil {
		return errors.Trace(err)
	}

	if app == nil {
		app = img.GetApp()
	}

	jid := c.Jid()
	if jid == 0 {
		return errors.New("Not started")
	}

	user := app.User
	if user == "" {
		user = "root"
	}

	args := []string{
		"-jid", strconv.Itoa(jid),
		"-user", user,
		"-group", app.Group,
		"-name", string(img.Manifest.Name),
	}

	for k, v := range app.Environment {
		args = append(args, "-setenv", k+"="+v)
	}

	args = append(args, app.Exec...)

	// FIXME:libexec
	return run.Command("/home/japhy/Go/src/github.com/3ofcoins/jetpack/bin/stage2", args...).Run()
}

type ContainerSlice []*Container

func (cc ContainerSlice) Len() int { return len(cc) }
func (cc ContainerSlice) Less(i, j int) bool {
	return bytes.Compare(cc[i].Manifest.UUID[:], cc[j].Manifest.UUID[:]) < 0
}
func (cc ContainerSlice) Swap(i, j int) { cc[i], cc[j] = cc[j], cc[i] }

func (cc ContainerSlice) Table() [][]string {
	rows := make([][]string, len(cc)+1)
	rows[0] = []string{"UUID", "IMAGE", "APP", "IP", "STATUS"}
	for i, c := range cc {
		imageID := ""
		if img, err := c.GetImage(); err != nil {
			imageID = fmt.Sprintf("[%v]", err)
		} else {
			imageID = img.UUID.String()
		}

		appName := ""
		if len(c.Manifest.Apps) > 0 {
			appName = string(c.Manifest.Apps[0].Name)
		}
		rows[i+1] = []string{
			c.Manifest.UUID.String(),
			imageID,
			appName,
			c.GetAnnotation("ip-address", ""),
			c.Status().String(),
		}
	}
	return rows
}
<|endoftext|>