<commit_before>package execstreamer

import (
	"errors"
	"os/exec"
	"strings"
)

func NewExecutorFromName(name string) (*executor, error) {
	cleanName := strings.ToLower(strings.TrimSpace(name))
	switch cleanName {
	case "winshell":
		return &executor{"cmd", []string{"/c"}}, nil
	case "bash":
		return &executor{"bash", []string{"-c"}}, nil
	default:
		return nil, errors.New("Executor not supported, name: " + name)
	}
}

type executor struct {
	exe  string
	args []string
}

func (e *executor) getFinalExeAndArgs(cmdExe string, cmdArgs ...string) (finalExe string, finalArgs []string) {
	if strings.TrimSpace(e.exe) == "" {
		return cmdExe, cmdArgs
	}

	finalExe = e.exe
	finalArgs = e.args
	finalArgs = append(finalArgs, cmdExe)
	finalArgs = append(finalArgs, cmdArgs...)
	return
}

func (e *executor) GetCommand(cmdExe string, cmdArgs ...string) *exec.Cmd {
	finalExe, finalArgs := e.getFinalExeAndArgs(cmdExe, cmdArgs...)

	return exec.Command(finalExe, finalArgs...)
}
<commit_msg>Add "none" executor<commit_after>package execstreamer

import (
	"errors"
	"os/exec"
	"strings"
)

func NewExecutorFromName(name string) (*executor, error) {
	cleanName := strings.ToLower(strings.TrimSpace(name))
	switch cleanName {
	case "winshell":
		return &executor{"cmd", []string{"/c"}}, nil
	case "bash":
		return &executor{"bash", []string{"-c"}}, nil
	case "none":
		return &executor{}, nil
	default:
		return nil, errors.New("Executor not supported, name: " + name)
	}
}

type executor struct {
	exe  string
	args []string
}

func (e *executor) getFinalExeAndArgs(cmdExe string, cmdArgs ...string) (finalExe string, finalArgs []string) {
	if strings.TrimSpace(e.exe) == "" {
		return cmdExe, cmdArgs
	}

	finalExe = e.exe
	finalArgs = e.args
	finalArgs = append(finalArgs, cmdExe)
	finalArgs = append(finalArgs, cmdArgs...)
	return
}

func (e *executor) GetCommand(cmdExe string, cmdArgs ...string) *exec.Cmd {
	finalExe, finalArgs := e.getFinalExeAndArgs(cmdExe, cmdArgs...)

	return exec.Command(finalExe, finalArgs...)
}
<|endoftext|>


<commit_before>package plugins

import (
	"fmt"
	"testing"
)

func TestBeer(t *testing.T) {
	beer := &Beer{}
	b, err := beer.get("oskar blues")
	if err != nil {
		t.Errorf("%+v\n", err)
	}

	if b.Nhits == 0 {
		t.Errorf("Expected 7 results; got %d\n", b.Nhits)
	}
	fmt.Printf("%+v", b)
}
<commit_msg>don't puke info on test<commit_after>package plugins

import (
	"testing"
)

func TestBeer(t *testing.T) {
	beer := &Beer{}
	b, err := beer.get("oskar blues")
	if err != nil {
		t.Errorf("%+v\n", err)
	}

	if b.Nhits == 0 {
		t.Errorf("Expected 7 results; got %d\n", b.Nhits)
	}
}
<|endoftext|>


<commit_before>package lazy

import (
	"os"
	"text/template"
)

const OS_INSTALL_TMPL = `{
  "id": "coreos-install",
  "name": "CoreOS Install",
  "profile": "install-reboot",
  "metadata": {
    "coreos_channel": "{{.Channel}}",
    "coreos_version": "{{.Version}}",
    "ignition_endpoint": "{{.M.URL}}/ignition",
    "baseurl": "{{.M.URL}}/assets/coreos"
  }
}
`

const K8S_CONTROLLER_TMPL = `{
  "id": "{{.ID}}",
  "name": "k8s controller",
  "profile": "k8s-controller",
  "selector": {
    "mac": "{{index .MAC 0}}",
    "os": "installed"
  },
  "metadata": {
    "container_runtime": "docker",
    "domain_name": "{{.Domain}}",
    "etcd_initial_cluster": "{{.InitialCluster}}",
    "etcd_name": "{{.ID}}",
    "k8s_cert_endpoint": "{{.M.URL}}/assets",
    "k8s_dns_service_ip": "10.3.0.10",
    "k8s_etcd_endpoints": "{{.Endpoints}}",
    "k8s_pod_network": "10.2.0.0/16",
    "k8s_service_ip_range": "10.3.0.0/24",
    "vip": {{with .VIP}}{{ . }}{{ end }},
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const K8S_WORKER_TMPL = `{
  "id": "{{.ID}}",
  "name": "k8s worker",
  "profile": "k8s-worker",
  "selector": {
    "mac": "{{index .MAC 0}}",
    "os": "installed"
  },
  "metadata": {
    "container_runtime": "docker",
    "domain_name": "{{.Domain}}",
    "etcd_initial_cluster": "{{.InitialCluster}}",
    "k8s_controller_endpoint": "{{.ControllerEndpoint}}",
    "k8s_cert_endpoint": "{{.M.URL}}/assets",
    "k8s_dns_service_ip": "10.3.0.10",
    "k8s_etcd_endpoints": "{{.Endpoints}}",
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const NODE_TMPL = `{
  "id": "{{.ID}}",
  "name": "Node {{.ID}}",
  "profile": "{{.Profile}}",
  "selector": {
    "mac": "{{index .MAC 0}}"
  },
  "metadata": {
    "domain_name": "{{.Domain}}",
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const DNSMASQ_TMPL = `# dnsmasq.conf

dhcp-option=3,{{.N.Gateway}}

{{ with (.Cls.GetKeepIPRange) }}
dhcp-range={{.Start}},{{.End}}{{ end }}

{{ range $i, $node := .Nodes }}{{ range $index, $mac := .MAC }}{{if lt $index (len $node.IP) }}
dhcp-host={{$mac}},{{index $node.IP $index}},1h{{ end }}{{ end }}{{ end }}

enable-tftp
tftp-root=/var/lib/tftpboot

dhcp-userclass=set:ipxe,iPXE
dhcp-boot=tag:#ipxe,undionly.kpxe
dhcp-boot=tag:ipxe,{{.M.URL}}/boot.ipxe

log-queries
log-dhcp

address=/{{.M.Domain}}/{{.M.IP}}

##### vip address #####
{{with .V }}{{if .Enable }}
address=/{{.Domain}}/{{.VIP}}{{end}}{{end}}

##### node address #####
{{range .Nodes}}{{if gt (len .IP) 0}}
address=/{{.Domain}}/{{index .IP 0}}{{end}}{{end}}

##### dns server #####
{{with .D}}{{range .DNS}}
server={{.}}{{end}}{{end}}
`

func writeTemplateToFile(tmplContent, name, fileName string, data interface{}) error {
	tmpl, err := template.New(name).Parse(tmplContent)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer f.Close()

	return tmpl.Execute(f, data)
}
<commit_msg>Refine dhcp template make it readable<commit_after>package lazy

import (
	"os"
	"text/template"
)

const OS_INSTALL_TMPL = `{
  "id": "coreos-install",
  "name": "CoreOS Install",
  "profile": "install-reboot",
  "metadata": {
    "coreos_channel": "{{.Channel}}",
    "coreos_version": "{{.Version}}",
    "ignition_endpoint": "{{.M.URL}}/ignition",
    "baseurl": "{{.M.URL}}/assets/coreos"
  }
}
`

const K8S_CONTROLLER_TMPL = `{
  "id": "{{.ID}}",
  "name": "k8s controller",
  "profile": "k8s-controller",
  "selector": {
    "mac": "{{index .MAC 0}}",
    "os": "installed"
  },
  "metadata": {
    "container_runtime": "docker",
    "domain_name": "{{.Domain}}",
    "etcd_initial_cluster": "{{.InitialCluster}}",
    "etcd_name": "{{.ID}}",
    "k8s_cert_endpoint": "{{.M.URL}}/assets",
    "k8s_dns_service_ip": "10.3.0.10",
    "k8s_etcd_endpoints": "{{.Endpoints}}",
    "k8s_pod_network": "10.2.0.0/16",
    "k8s_service_ip_range": "10.3.0.0/24",
    "vip": {{with .VIP}}{{ . }}{{ end }},
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const K8S_WORKER_TMPL = `{
  "id": "{{.ID}}",
  "name": "k8s worker",
  "profile": "k8s-worker",
  "selector": {
    "mac": "{{index .MAC 0}}",
    "os": "installed"
  },
  "metadata": {
    "container_runtime": "docker",
    "domain_name": "{{.Domain}}",
    "etcd_initial_cluster": "{{.InitialCluster}}",
    "k8s_controller_endpoint": "{{.ControllerEndpoint}}",
    "k8s_cert_endpoint": "{{.M.URL}}/assets",
    "k8s_dns_service_ip": "10.3.0.10",
    "k8s_etcd_endpoints": "{{.Endpoints}}",
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const NODE_TMPL = `{
  "id": "{{.ID}}",
  "name": "Node {{.ID}}",
  "profile": "{{.Profile}}",
  "selector": {
    "mac": "{{index .MAC 0}}"
  },
  "metadata": {
    "domain_name": "{{.Domain}}",
    "interfaces": {{.Nics}},
    "ssh_authorized_keys": {{.AuthorizedKeys}}
  }
}
`

const DNSMASQ_TMPL = `# dnsmasq.conf

### DHCP CONFIG ###
{{- if .N.Gateway }}
dhcp-option=3,{{.N.Gateway}}
{{- end }}

{{- with (.Cls.GetKeepIPRange) }}
dhcp-range={{.Start}},{{.End}}
{{- end }}

{{- range $i, $node := .Nodes }}
  {{- range $index, $mac := .MAC }}
    {{- if lt $index (len $node.IP) }}
dhcp-host={{$mac}},{{index $node.IP $index}},1h
    {{- end }}
  {{- end }}
{{- end }}

dhcp-userclass=set:ipxe,iPXE
dhcp-boot=tag:#ipxe,undionly.kpxe
dhcp-boot=tag:ipxe,{{.M.URL}}/boot.ipxe

### TFTP CONFIG ###
enable-tftp
tftp-root=/var/lib/tftpboot

### DNS CONFIG ###

address=/{{.M.Domain}}/{{.M.IP}}

##### vip address #####
{{- with .V }}
  {{- if .Enable }}
address=/{{.Domain}}/{{.VIP}}
  {{- end }}
{{- end }}

##### node address #####
{{- range .Nodes }}
  {{- if gt (len .IP) 0 }}
address=/{{.Domain}}/{{index .IP 0}}
  {{- end }}
{{- end }}

##### dns server #####
{{- with .D }}
  {{- range .DNS }}
server={{.}}
  {{- end }}
{{- end }}

### OTHER CONFIG ###
log-queries
log-dhcp
`

func writeTemplateToFile(tmplContent, name, fileName string, data interface{}) error {
	tmpl, err := template.New(name).Parse(tmplContent)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(fileName, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer f.Close()

	return tmpl.Execute(f, data)
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	sanitizeHTML      = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
	normalizeFavicons = flag.Bool("normalizeFavicons", true, "If true, resize favicons to 256x256 and encode as PNG.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = makeBodyPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

func makeBodyPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("title", "alt").OnElements("img")
	return p
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")
	rss.TimeLayouts = append(rss.TimeLayouts, "Mon, 02 Jan 2006")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		// Some feeds give back content that is HTML-escaped. When this happens,
		// sanitization makes the content appear as raw, escaped text. There's not
		// a canonical way of determining if the content is given here as escaped
		// or not, so we use a heuristic.
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := maybeParseArticleContent(item.Link)

		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		summary = maybeRewriteImageSourceUrls(summary)

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err := tryIconFetch(f.Image.URL); err == nil {
		icon = i
	} else if i, err = tryIconFetch(f.Link); err == nil {
		icon = i
	} else if i, err = tryIconFetch(feedHost); err == nil {
		icon = i
	} else {
		log.V(2).Infof("Could not find suitable icon for feed: %s", feedHost)
		return
	}

	select {
	case send <- maybeResizeImage(feed.ID, icon):
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<commit_msg>core/fetch: Also do image proxying on the "content" field.<commit_after>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	sanitizeHTML      = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
	normalizeFavicons = flag.Bool("normalizeFavicons", true, "If true, resize favicons to 256x256 and encode as PNG.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = makeBodyPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

func makeBodyPolicy() *bluemonday.Policy {
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("title", "alt").OnElements("img")
	return p
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")
	rss.TimeLayouts = append(rss.TimeLayouts, "Mon, 02 Jan 2006")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			cancel()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		// Some feeds give back content that is HTML-escaped. When this happens,
		// sanitization makes the content appear as raw, escaped text. There's not
		// a canonical way of determining if the content is given here as escaped
		// or not, so we use a heuristic.
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := maybeParseArticleContent(item.Link)

		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		content = maybeRewriteImageSourceUrls(content)
		summary = maybeRewriteImageSourceUrls(summary)

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err := tryIconFetch(f.Image.URL); err == nil {
		icon = i
	} else if i, err = tryIconFetch(f.Link); err == nil {
		icon = i
	} else if i, err = tryIconFetch(feedHost); err == nil {
		icon = i
	} else {
		log.V(2).Infof("Could not find suitable icon for feed: %s", feedHost)
		return
	}

	select {
	case send <- maybeResizeImage(feed.ID, icon):
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<|endoftext|>


<commit_before>// Package flagutil provides a collection of types implementing the flag.Value interface.
package flagutil

import (
	"flag"
	"net"
	"net/url"
	"strings"
	"time"
)

// NetworkAddresses is a slice of string that have been validated as valid network addresses.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice and validated as valid network addresses.
type NetworkAddresses []string

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
// Additionally, each value is then passed to net.SplitHostPort for validation that it's a correct network address.
func (a *NetworkAddresses) Set(s string) error {
	for _, t := range strings.Split(s, ",") {
		_, _, err := net.SplitHostPort(t)
		if err != nil {
			return err
		}

		*a = append(*a, t)
	}

	return nil
}

// String implements the flag.Value interface. It returns the slice as a comma-separated string.
func (a NetworkAddresses) String() string { return strings.Join(a, ",") }

// StringSlice returns the network addresses as a slice of string.
func (a NetworkAddresses) StringSlice() []string {
	var res []string
	for _, el := range a {
		res = append(res, el)
	}
	return res
}

// Strings is a slice of string.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice.
type Strings []string

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
func (s *Strings) Set(str string) error {
	for _, t := range strings.Split(str, ",") {
		*s = append(*s, t)
	}

	return nil
}

// String implements the flag.Value interface. It returns the slice as a comma-separated string.
func (s Strings) String() string {
	return strings.Join(s, ",")
}

// URL is a wrapper around a url.URL type.
// Use it as a flag value when you want to validate a flag as a valid URL.
type URL struct {
	url.URL
	parsed bool
}

// Set implements the flag.Value interface. It parses the string as a url.URL value.
func (u *URL) Set(s string) error {
	url, err := url.Parse(s)
	if err != nil {
		return err
	}

	u.URL = *url
	u.parsed = true

	return nil
}

// String implements the flag.Value interface. It returns the underlying url.URL as a string.
func (u URL) String() string { return u.URL.String() }

// IsValid returns true if the URL object is valid, false otherwise. Valid here means it has correctly parsed an URL.
func (u URL) IsValid() bool { return u.parsed }

// URLs is a slice of url.URL.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice and validated as valid url.URL.
type URLs []url.URL

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
func (u *URLs) Set(s string) error {
	for _, t := range strings.Split(s, ",") {
		url, err := url.Parse(t)
		if err != nil {
			return err
		}

		*u = append(*u, *url)
	}

	return nil
}

// String implements the flag.Value interface. It returns the string representation of each url.URL as a comma-separated string.
func (u URLs) String() string {
	var r []string

	for _, url := range u {
		r = append(r, url.String())
	}

	return strings.Join(r, ",")
}

type Duration time.Duration

func (d *Duration) Set(s string) error {
	tmp, err := time.ParseDuration(s)
	if err != nil {
		return err
	}

	*d = Duration(tmp)

	return nil
}

func (d Duration) String() string { return time.Duration(d).String() }

var (
	_ flag.Value = (*NetworkAddresses)(nil)
	_ flag.Value = (*Strings)(nil)
	_ flag.Value = (*URLs)(nil)
	_ flag.Value = (*Duration)(nil)
)
<commit_msg>the default flag can already parse a time.Duration...<commit_after>// Package flagutil provides a collection of types implementing the flag.Value interface.
package flagutil

import (
	"flag"
	"net"
	"net/url"
	"strings"
)

// NetworkAddresses is a slice of string that have been validated as valid network addresses.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice and validated as valid network addresses.
type NetworkAddresses []string

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
// Additionally, each value is then passed to net.SplitHostPort for validation that it's a correct network address.
func (a *NetworkAddresses) Set(s string) error {
	for _, t := range strings.Split(s, ",") {
		_, _, err := net.SplitHostPort(t)
		if err != nil {
			return err
		}

		*a = append(*a, t)
	}

	return nil
}

// String implements the flag.Value interface. It returns the slice as a comma-separated string.
func (a NetworkAddresses) String() string { return strings.Join(a, ",") }

// StringSlice returns the network addresses as a slice of string.
func (a NetworkAddresses) StringSlice() []string {
	var res []string
	for _, el := range a {
		res = append(res, el)
	}
	return res
}

// Strings is a slice of string.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice.
type Strings []string

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
func (s *Strings) Set(str string) error {
	for _, t := range strings.Split(str, ",") {
		*s = append(*s, t)
	}

	return nil
}

// String implements the flag.Value interface. It returns the slice as a comma-separated string.
func (s Strings) String() string {
	return strings.Join(s, ",")
}

// URL is a wrapper around a url.URL type.
// Use it as a flag value when you want to validate a flag as a valid URL.
type URL struct {
	url.URL
	parsed bool
}

// Set implements the flag.Value interface. It parses the string as a url.URL value.
func (u *URL) Set(s string) error {
	url, err := url.Parse(s)
	if err != nil {
		return err
	}

	u.URL = *url
	u.parsed = true

	return nil
}

// String implements the flag.Value interface. It returns the underlying url.URL as a string.
func (u URL) String() string { return u.URL.String() }

// IsValid returns true if the URL object is valid, false otherwise. Valid here means it has correctly parsed an URL.
func (u URL) IsValid() bool { return u.parsed }

// URLs is a slice of url.URL.
// Use it as a flag value when you want to pass a comma-separated list of strings to a flag
// and have it to be automatically parsed into a slice and validated as valid url.URL.
type URLs []url.URL

// Set implements the flag.Value interface. It parses the string as a comma-separated string.
func (u *URLs) Set(s string) error {
	for _, t := range strings.Split(s, ",") {
		url, err := url.Parse(t)
		if err != nil {
			return err
		}

		*u = append(*u, *url)
	}

	return nil
}

// String implements the flag.Value interface. It returns the string representation of each url.URL as a comma-separated string.
func (u URLs) String() string {
	var r []string

	for _, url := range u {
		r = append(r, url.String())
	}

	return strings.Join(r, ",")
}

var (
	_ flag.Value = (*NetworkAddresses)(nil)
	_ flag.Value = (*Strings)(nil)
	_ flag.Value = (*URLs)(nil)
)
<|endoftext|>


<commit_before>package nstcc

import (
	"bytes"
	"errors"
)

func dup(s []byte) []byte {
	t := make([]byte, len(s))
	copy(t, s)
	return t
}

const (
	hashInit = 1
	hashSize = 8192
)

func hashFunc(h uint32, c uint8) uint32 {
	return h*263 + uint32(c)
}

type idents struct {
	list []*tokenSym
	hash [hashSize]*tokenSym
}

// The TCC code calls this tok_alloc.
func (m *idents) byStr(s []byte) (*tokenSym, error) {
	h := uint32(hashInit)
	for _, c := range s {
		h = hashFunc(h, c)
	}
	p := &m.hash[h&(hashSize-1)]
	for {
		t := *p
		if t == nil {
			break
		}
		if bytes.Equal(s, t.str) {
			return t, nil
		}
		p = &t.hashNext
	}
	return m.alloc(p, s)
}

// The TCC code calls this tok_alloc_new.
func (m *idents) alloc(p **tokenSym, s []byte) (*tokenSym, error) {
	tok := tokIdent + token(len(m.list))
	if tok >= symFirstAnom {
		return nil, errors.New("nstcc: memory full")
	}
	t := &tokenSym{
		tok: tok,
		str: dup(s),
	}
	m.list = append(m.list, t)
	*p = t
	return t, nil
}

// tokIdent is the token value of the first identifier token. Token values
// greater than or equal to tokIdent represent identifiers. Token values less
// than tokIdent represent symbols.
const tokIdent = 256

type token int32

type tokenSym struct {
	hashNext *tokenSym
	// TODO: symThis, symThat.
	tok token
	str []byte
}

const symFirstAnom = 0x10000000 // First anonymous sym.

type sym struct {
	// TODO.
}
<commit_msg>Add an idents.byTok method.<commit_after>package nstcc

import (
	"bytes"
	"errors"
)

func dup(s []byte) []byte {
	t := make([]byte, len(s))
	copy(t, s)
	return t
}

const (
	hashInit = 1
	hashSize = 8192
)

func hashFunc(h uint32, c uint8) uint32 {
	return h*263 + uint32(c)
}

type idents struct {
	list []*tokenSym
	hash [hashSize]*tokenSym
}

func (m *idents) byTok(t token) *tokenSym {
	return m.list[t-tokIdent]
}

// The TCC code calls this tok_alloc.
func (m *idents) byStr(s []byte) (*tokenSym, error) {
	h := uint32(hashInit)
	for _, c := range s {
		h = hashFunc(h, c)
	}
	p := &m.hash[h&(hashSize-1)]
	for {
		y := *p
		if y == nil {
			break
		}
		if bytes.Equal(s, y.str) {
			return y, nil
		}
		p = &y.hashNext
	}
	return m.alloc(p, s)
}

// The TCC code calls this tok_alloc_new.
func (m *idents) alloc(p **tokenSym, s []byte) (*tokenSym, error) {
	t := tokIdent + token(len(m.list))
	if t >= symFirstAnom {
		return nil, errors.New("nstcc: memory full")
	}
	y := &tokenSym{
		tok: t,
		str: dup(s),
	}
	m.list = append(m.list, y)
	*p = y
	return y, nil
}

// tokIdent is the token value of the first identifier token. Token values
// greater than or equal to tokIdent represent identifiers. Token values less
// than tokIdent represent symbols.
const tokIdent = 256

type token int32

type tokenSym struct {
	hashNext *tokenSym
	// TODO: symThis, symThat.
	tok token
	str []byte
}

const symFirstAnom = 0x10000000 // First anonymous sym.

type sym struct {
	// TODO.
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import "fmt"

// Token is the set of lexical tokens.
type Token int

// The list of all possible tokens.
const (
	ILLEGAL Token = -iota
	EOF
	LIT

	IF
	THEN
	ELIF
	ELSE
	FI
	WHILE
	FOR
	IN
	UNTIL
	DO
	DONE
	CASE
	ESAC

	AND  // &
	LAND // &&
	OR   // |
	LOR  // ||

	BANG    // !
	EXP     // $
	LPAREN  // (
	LBRACE  // {
	DLPAREN // ((

	RPAREN     // )
	RBRACE     // }
	DRPAREN    // ))
	SEMICOLON  // ;
	DSEMICOLON // ;;

	RDRIN    // <
	RDROUT   // >
	OPRDWR   // <>
	DPLIN    // <&
	DPLOUT   // >&
	APPEND   // >>
	HEREDOC  // <<
	DHEREDOC // <<-
)

// Pos is the internal representation of a position within a source
// file.
// TODO: replace struct with a byte offset
type Pos struct {
	Line   int // line number, starting at 1
	Column int // column number, starting at 1
}

func (p Pos) IsValid() bool { return p.Line > 0 && p.Column > 0 }

// Position describes an arbitrary position in a source file. Offsets,
// including column numbers, are in bytes.
type Position struct {
	Filename string
	Line     int // line number, starting at 1
	Column   int // column number, starting at 1
}

func (p Position) String() string {
	if p.Filename == "" {
		return fmt.Sprintf("%d:%d", p.Line, p.Column)
	}
	return fmt.Sprintf("%s:%d:%d", p.Filename, p.Line, p.Column)
}

func init() {
	for _, t := range tokList {
		tokNames[t.tok] = t.str
	}
}

var (
	tokNames = map[Token]string{
		ILLEGAL: `ILLEGAL`,
		EOF:     `EOF`,
		LIT:     `literal`,

		IF:    "if",
		THEN:  "then",
		ELIF:  "elif",
		ELSE:  "else",
		FI:    "fi",
		WHILE: "while",
		FOR:   "for",
		IN:    "in",
		UNTIL: "until",
		DO:    "do",
		DONE:  "done",
		CASE:  "case",
		ESAC:  "esac",
	}

	tokList = [...]struct {
		str string
		tok Token
	}{
		{"&", AND},
		{"&&", LAND},
		{"|", OR},
		{"||", LOR},

		{"!", BANG},
		{"$", EXP},
		{"(", LPAREN},
		{"{", LBRACE},
		{"((", DLPAREN},

		{")", RPAREN},
		{"}", RBRACE},
		{"))", DRPAREN},
		{";", SEMICOLON},
		{";;", DSEMICOLON},

		{"<", RDRIN},
		{">", RDROUT},
		{"<>", OPRDWR},
		{"<&", DPLIN},
		{">&", DPLOUT},
		{">>", APPEND},
		{"<<", HEREDOC},
		{"<<-", DHEREDOC},
	}
)

func (t Token) String() string {
	if s, e := tokNames[t]; e {
		return s
	}
	return string(t)
}

func doToken(readOnly func(string) bool, readByte func() (byte, error)) (Token, error) {
	// In reverse, to not treat e.g. && as & two times
	for i := len(tokList) - 1; i >= 0; i-- {
		t := tokList[i]
		if readOnly(t.str) {
			return t.tok, nil
		}
	}
	b, err := readByte()
	return Token(b), err
}
<commit_msg>Remove TODO from token.Pos<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package sh

import "fmt"

// Token is the set of lexical tokens.
type Token int

// The list of all possible tokens.
const (
	ILLEGAL Token = -iota
	EOF
	LIT

	IF
	THEN
	ELIF
	ELSE
	FI
	WHILE
	FOR
	IN
	UNTIL
	DO
	DONE
	CASE
	ESAC

	AND  // &
	LAND // &&
	OR   // |
	LOR  // ||

	BANG    // !
	EXP     // $
	LPAREN  // (
	LBRACE  // {
	DLPAREN // ((

	RPAREN     // )
	RBRACE     // }
	DRPAREN    // ))
	SEMICOLON  // ;
	DSEMICOLON // ;;

	RDRIN    // <
	RDROUT   // >
	OPRDWR   // <>
	DPLIN    // <&
	DPLOUT   // >&
	APPEND   // >>
	HEREDOC  // <<
	DHEREDOC // <<-
)

// Pos is the internal representation of a position within a source
// file.
type Pos struct {
	Line   int // line number, starting at 1
	Column int // column number, starting at 1
}

func (p Pos) IsValid() bool { return p.Line > 0 && p.Column > 0 }

// Position describes an arbitrary position in a source file. Offsets,
// including column numbers, are in bytes.
type Position struct {
	Filename string
	Line     int // line number, starting at 1
	Column   int // column number, starting at 1
}

func (p Position) String() string {
	if p.Filename == "" {
		return fmt.Sprintf("%d:%d", p.Line, p.Column)
	}
	return fmt.Sprintf("%s:%d:%d", p.Filename, p.Line, p.Column)
}

func init() {
	for _, t := range tokList {
		tokNames[t.tok] = t.str
	}
}

var (
	tokNames = map[Token]string{
		ILLEGAL: `ILLEGAL`,
		EOF:     `EOF`,
		LIT:     `literal`,

		IF:    "if",
		THEN:  "then",
		ELIF:  "elif",
		ELSE:  "else",
		FI:    "fi",
		WHILE: "while",
		FOR:   "for",
		IN:    "in",
		UNTIL: "until",
		DO:    "do",
		DONE:  "done",
		CASE:  "case",
		ESAC:  "esac",
	}

	tokList = [...]struct {
		str string
		tok Token
	}{
		{"&", AND},
		{"&&", LAND},
		{"|", OR},
		{"||", LOR},

		{"!", BANG},
		{"$", EXP},
		{"(", LPAREN},
		{"{", LBRACE},
		{"((", DLPAREN},

		{")", RPAREN},
		{"}", RBRACE},
		{"))", DRPAREN},
		{";", SEMICOLON},
		{";;", DSEMICOLON},

		{"<", RDRIN},
		{">", RDROUT},
		{"<>", OPRDWR},
		{"<&", DPLIN},
		{">&", DPLOUT},
		{">>", APPEND},
		{"<<", HEREDOC},
		{"<<-", DHEREDOC},
	}
)

func (t Token) String() string {
	if s, e := tokNames[t]; e {
		return s
	}
	return string(t)
}

func doToken(readOnly func(string) bool, readByte func() (byte, error)) (Token, error) {
	// In reverse, to not treat e.g. && as & two times
	for i := len(tokList) - 1; i >= 0; i-- {
		t := tokList[i]
		if readOnly(t.str) {
			return t.tok, nil
		}
	}
	b, err := readByte()
	return Token(b), err
}
<|endoftext|>


<commit_before>package dom

// Auto-generated by internal/cmd/genwrapnode/genwrapnode.go. DO NOT EDIT!

import (
	"fmt"

	"github.com/lestrrat/go-libxml2/clib"
	"github.com/lestrrat/go-libxml2/types"
)

func wrapNamespaceNode(ptr uintptr) *Namespace {
	var n Namespace
	n.ptr = ptr
	return &n
}

func wrapAttributeNode(ptr uintptr) *Attribute {
	var n Attribute
	n.ptr = ptr
	return &n
}

func wrapCDataSectionNode(ptr uintptr) *CDataSection {
	var n CDataSection
	n.ptr = ptr
	return &n
}

func wrapCommentNode(ptr uintptr) *Comment {
	var n Comment
	n.ptr = ptr
	return &n
}

func wrapElementNode(ptr uintptr) *Element {
	var n Element
	n.ptr = ptr
	return &n
}

func wrapTextNode(ptr uintptr) *Text {
	var n Text
	n.ptr = ptr
	return &n
}

func wrapPiNode(ptr uintptr) *Pi {
	var n Pi
	n.ptr = ptr
	return &n
}

// WrapNode is a function created with the sole purpose of allowing
// go-libxml2 consumers that can generate a C.xmlNode pointer to
// create libxml2.Node types, e.g. go-xmlsec.
func WrapNode(n uintptr) (types.Node, error) {
	switch typ := clib.XMLGetNodeTypeRaw(n); typ {
	case clib.AttributeNode:
		return wrapAttributeNode(n), nil
	case clib.CDataSectionNode:
		return wrapCDataSectionNode(n), nil
	case clib.CommentNode:
		return wrapCommentNode(n), nil
	case clib.ElementNode:
		return wrapElementNode(n), nil
	case clib.TextNode:
		return wrapTextNode(n), nil
	case clib.PiNode:
		return wrapPiNode(n), nil
	default:
		return nil, fmt.Errorf("unknown node: %%d", typ)
	}
}
<commit_msg>fix %%d<commit_after>package dom

// Auto-generated by internal/cmd/genwrapnode/genwrapnode.go. DO NOT EDIT!

import (
	"fmt"

	"github.com/lestrrat/go-libxml2/clib"
	"github.com/lestrrat/go-libxml2/types"
)

func wrapNamespaceNode(ptr uintptr) *Namespace {
	var n Namespace
	n.ptr = ptr
	return &n
}

func wrapAttributeNode(ptr uintptr) *Attribute {
	var n Attribute
	n.ptr = ptr
	return &n
}

func wrapCDataSectionNode(ptr uintptr) *CDataSection {
	var n CDataSection
	n.ptr = ptr
	return &n
}

func wrapCommentNode(ptr uintptr) *Comment {
	var n Comment
	n.ptr = ptr
	return &n
}

func wrapElementNode(ptr uintptr) *Element {
	var n Element
	n.ptr = ptr
	return &n
}

func wrapTextNode(ptr uintptr) *Text {
	var n Text
	n.ptr = ptr
	return &n
}

func wrapPiNode(ptr uintptr) *Pi {
	var n Pi
	n.ptr = ptr
	return &n
}

// WrapNode is a function created with the sole purpose of allowing
// go-libxml2 consumers that can generate a C.xmlNode pointer to
// create libxml2.Node types, e.g. go-xmlsec.
func WrapNode(n uintptr) (types.Node, error) {
	switch typ := clib.XMLGetNodeTypeRaw(n); typ {
	case clib.AttributeNode:
		return wrapAttributeNode(n), nil
	case clib.CDataSectionNode:
		return wrapCDataSectionNode(n), nil
	case clib.CommentNode:
		return wrapCommentNode(n), nil
	case clib.ElementNode:
		return wrapElementNode(n), nil
	case clib.TextNode:
		return wrapTextNode(n), nil
	case clib.PiNode:
		return wrapPiNode(n), nil
	default:
		return nil, fmt.Errorf("unknown node: %d", typ)
	}
}
<|endoftext|>


<commit_before>package commands_test

import (
	"errors"
	"fmt"
	"github.com/concourse/concourse/fly/commands/internal/flaghelpers"
	"strconv"

	"github.com/concourse/concourse/atc"
	. "github.com/concourse/concourse/fly/commands"
	"github.com/concourse/concourse/go-concourse/concourse"
	fakes "github.com/concourse/concourse/go-concourse/concourse/concoursefakes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Helper Functions", func() {
	Describe("#GetBuild", func() {
		var client *fakes.FakeClient
		var team *fakes.FakeTeam
		expectedBuildID := "123"
		expectedBuildName := "5"
		expectedJobName := "myjob"
		expectedPipelineName := "mypipeline"
		expectedBuild := atc.Build{
			ID:      123,
			Name:    expectedBuildName,
			Status:  "Great Success",
			JobName: expectedJobName,
			APIURL:  fmt.Sprintf("api/v1/builds/%s", expectedBuildID),
		}

		BeforeEach(func() {
			client = new(fakes.FakeClient)
			team = new(fakes.FakeTeam)
		})

		Context("when passed a build id", func() {
			Context("when build exists", func() {
				BeforeEach(func() {
					client.BuildReturns(expectedBuild, true, nil)
				})

				It("returns the build", func() {
					build, err := GetBuild(client, nil, "", expectedBuildID, "")
					Expect(err).NotTo(HaveOccurred())
					Expect(build).To(Equal(expectedBuild))
					Expect(client.BuildCallCount()).To(Equal(1))
					Expect(client.BuildArgsForCall(0)).To(Equal(expectedBuildID))
				})
			})

			Context("when a build does not exist", func() {
				BeforeEach(func() {
					client.BuildReturns(atc.Build{}, false, nil)
				})

				It("returns an error", func() {
					_, err := GetBuild(client, nil, "", expectedBuildID, "")
					Expect(err).To(MatchError("build not found"))
				})
			})
		})

		Context("when passed a pipeline and job name", func() {
			Context("when job exists", func() {
				Context("when the next build exists", func() {
					BeforeEach(func() {
						job := atc.Job{
							Name:      expectedJobName,
							NextBuild: &expectedBuild,
						}
						team.JobReturns(job, true, nil)
					})

					It("returns the next build for that job", func() {
						build, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
						Expect(err).NotTo(HaveOccurred())
						Expect(build).To(Equal(expectedBuild))
						Expect(team.JobCallCount()).To(Equal(1))
						pipelineName, jobName := team.JobArgsForCall(0)
						Expect(pipelineName).To(Equal(expectedPipelineName))
						Expect(jobName).To(Equal(expectedJobName))
					})
				})

				Context("when the only the finished build exists", func() {
					BeforeEach(func() {
						job := atc.Job{
							Name:          expectedJobName,
							FinishedBuild: &expectedBuild,
						}
						team.JobReturns(job, true, nil)
					})

					It("returns the finished build for that job", func() {
						build, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
						Expect(err).NotTo(HaveOccurred())
						Expect(build).To(Equal(expectedBuild))
						Expect(team.JobCallCount()).To(Equal(1))
						pipelineName, jobName := team.JobArgsForCall(0)
						Expect(pipelineName).To(Equal(expectedPipelineName))
						Expect(jobName).To(Equal(expectedJobName))
					})
				})

				Context("when no builds exist", func() {
					BeforeEach(func() {
						job := atc.Job{
							Name: expectedJobName,
						}
						team.JobReturns(job, true, nil)
					})

					It("returns an error", func() {
						_, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
						Expect(err).To(HaveOccurred())
					})
				})
			})

			Context("when job does not exists", func() {
				BeforeEach(func() {
					team.JobReturns(atc.Job{}, false, nil)
				})

				It("returns an error", func() {
					_, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
					Expect(err).To(MatchError("job not found"))
				})
			})
		})

		Context("when passed pipeline, job, and build names", func() {
			Context("when the build exists", func() {
				BeforeEach(func() {
					team.JobBuildReturns(expectedBuild, true, nil)
				})

				It("returns the build", func() {
					build, err := GetBuild(client, team, expectedJobName, expectedBuildName, expectedPipelineName)
					Expect(err).NotTo(HaveOccurred())
					Expect(build).To(Equal(expectedBuild))
					Expect(team.JobBuildCallCount()).To(Equal(1))
					pipelineName, jobName, buildName := team.JobBuildArgsForCall(0)
					Expect(pipelineName).To(Equal(expectedPipelineName))
					Expect(buildName).To(Equal(expectedBuildName))
					Expect(jobName).To(Equal(expectedJobName))
				})
			})

			Context("when the build does not exist", func() {
				BeforeEach(func() {
					team.JobBuildReturns(atc.Build{}, false, nil)
				})

				It("returns an error", func() {
					_, err := GetBuild(client, team, expectedJobName, expectedBuildName, expectedPipelineName)
					Expect(err).To(MatchError("build not found"))
				})
			})
		})

		Context("when nothing is passed", func() {
			var allBuilds [300]atc.Build

			expectedOneOffBuild := atc.Build{
				ID:      150,
				Name:    expectedBuildName,
				Status:  "success",
				JobName: "",
				APIURL:  fmt.Sprintf("api/v1/builds/%s", expectedBuildID),
			}

			BeforeEach(func() {
				for i := 300 - 1; i >= 0; i-- {
					allBuilds[i] = atc.Build{
						ID:      i,
						Name:    strconv.Itoa(i),
						JobName: "some-job",
						APIURL:  fmt.Sprintf("api/v1/builds/%d", i),
					}
				}

				allBuilds[150] = expectedOneOffBuild

				client.BuildsStub = func(page concourse.Page) ([]atc.Build, concourse.Pagination, error) {
					var builds []atc.Build
					if page.Since != 0 {
						builds = allBuilds[page.Since : page.Since+page.Limit]
					} else {
						builds = allBuilds[0:page.Limit]
					}

					pagination := concourse.Pagination{
						Previous: &concourse.Page{
							Limit: page.Limit,
							Until: builds[0].ID,
						},
						Next: &concourse.Page{
							Limit: page.Limit,
							Since: builds[len(builds)-1].ID,
						},
					}

					return builds, pagination, nil
				}
			})

			It("returns latest one off build", func() {
				build, err := GetBuild(client, nil, "", "", "")
				Expect(err).NotTo(HaveOccurred())
				Expect(build).To(Equal(expectedOneOffBuild))
				Expect(client.BuildsCallCount()).To(Equal(2))
			})
		})
	})
	Describe("#GetLatestResourceVersions", func() {
		var team *fakes.FakeTeam
		var resourceVersions []atc.ResourceVersion

		resource := flaghelpers.ResourceFlag{
			PipelineName: "mypipeline",
			ResourceName: "myresource",
		}

		BeforeEach(func() {
			team = new(fakes.FakeTeam)
			resourceVersions = []atc.ResourceVersion{
				{
					ID:      1,
					Version: atc.Version{"version": "v1"},
				},
				{
					ID:      2,
					Version: atc.Version{"version": "v1"},
				},
			}
		})

		When("resource versions exist", func() {
			It("returns latest resource version", func() {
				team.ResourceVersionsReturns(resourceVersions, concourse.Pagination{}, true, nil)
				latestResourceVersion, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v1"})
				Expect(err).NotTo(HaveOccurred())
				Expect(latestResourceVersion.Version).To(Equal(atc.Version{"version": "v1"}))
				Expect(latestResourceVersion.ID).To(Equal(1))
			})
		})

		When("call to resource versions returns an error", func() {
			It("returns an error", func() {
				team.ResourceVersionsReturns(nil, concourse.Pagination{}, false, errors.New("fake error"))
				_, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v1"})
				Expect(err).To(MatchError("fake error"))
			})
		})

		When("call to resource versions returns an empty array", func() {
			It("returns an error", func() {
				team.ResourceVersionsReturns([]atc.ResourceVersion{}, concourse.Pagination{}, true, nil)
				_, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v2"})
				Expect(err).To(MatchError("could not find version matching {\"version\":\"v2\"}"))
			})
		})
	})
})
<commit_msg>Increased coverage for atc/helpers GetBuild<commit_after>package commands_test

import (
	"errors"
	"fmt"
	"github.com/concourse/concourse/fly/commands/internal/flaghelpers"
	"strconv"

	"github.com/concourse/concourse/atc"
	. "github.com/concourse/concourse/fly/commands"
	"github.com/concourse/concourse/go-concourse/concourse"
	fakes "github.com/concourse/concourse/go-concourse/concourse/concoursefakes"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Helper Functions", func() {
	Describe("#GetBuild", func() {
		var client *fakes.FakeClient
		var team *fakes.FakeTeam
		expectedBuildID := "123"
		expectedBuildName := "5"
		expectedJobName := "myjob"
		expectedPipelineName := "mypipeline"
		expectedBuild := atc.Build{
			ID:      123,
			Name:    expectedBuildName,
			Status:  "Great Success",
			JobName: expectedJobName,
			APIURL:  fmt.Sprintf("api/v1/builds/%s", expectedBuildID),
		}

		BeforeEach(func() {
			client = new(fakes.FakeClient)
			team = new(fakes.FakeTeam)
		})

		Context("when passed a build id", func() {
			Context("when no error is encountered while fetching build", func() {
				Context("when build exists", func() {
					BeforeEach(func() {
						client.BuildReturns(expectedBuild, true, nil)
					})

					It("returns the build", func() {
						build, err := GetBuild(client, nil, "", expectedBuildID, "")
						Expect(err).NotTo(HaveOccurred())
						Expect(build).To(Equal(expectedBuild))
						Expect(client.BuildCallCount()).To(Equal(1))
						Expect(client.BuildArgsForCall(0)).To(Equal(expectedBuildID))
					})
				})

				Context("when a build does not exist", func() {
					BeforeEach(func() {
						client.BuildReturns(atc.Build{}, false, nil)
					})

					It("returns an error", func() {
						_, err := GetBuild(client, nil, "", expectedBuildID, "")
						Expect(err).To(HaveOccurred())
						Expect(err).To(MatchError("build not found"))
					})
				})
			})

			Context("when an error is encountered while fetching build", func() {
				BeforeEach(func() {
					client.BuildReturns(atc.Build{}, false, errors.New("some-error"))
				})

				It("return an error", func() {
					_, err := GetBuild(client, nil, "", expectedBuildID, "")
					Expect(err).To(MatchError("failed to get build some-error"))
				})
			})
		})

		Context("when passed a pipeline and job name", func() {
			Context("when no error was encountered while looking up for team job", func() {
				Context("when job exists", func() {
					Context("when the next build exists", func() {
						BeforeEach(func() {
							job := atc.Job{
								Name:      expectedJobName,
								NextBuild: &expectedBuild,
							}
							team.JobReturns(job, true, nil)
						})

						It("returns the next build for that job", func() {
							build, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
							Expect(err).NotTo(HaveOccurred())
							Expect(build).To(Equal(expectedBuild))
							Expect(team.JobCallCount()).To(Equal(1))
							pipelineName, jobName := team.JobArgsForCall(0)
							Expect(pipelineName).To(Equal(expectedPipelineName))
							Expect(jobName).To(Equal(expectedJobName))
						})
					})

					Context("when the only the finished build exists", func() {
						BeforeEach(func() {
							job := atc.Job{
								Name:          expectedJobName,
								FinishedBuild: &expectedBuild,
							}
							team.JobReturns(job, true, nil)
						})

						It("returns the finished build for that job", func() {
							build, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
							Expect(err).NotTo(HaveOccurred())
							Expect(build).To(Equal(expectedBuild))
							Expect(team.JobCallCount()).To(Equal(1))
							pipelineName, jobName := team.JobArgsForCall(0)
							Expect(pipelineName).To(Equal(expectedPipelineName))
							Expect(jobName).To(Equal(expectedJobName))
						})
					})

					Context("when no builds exist", func() {
						BeforeEach(func() {
							job := atc.Job{
								Name: expectedJobName,
							}
							team.JobReturns(job, true, nil)
						})

						It("returns an error", func() {
							_, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
							Expect(err).To(HaveOccurred())
						})
					})
				})

				Context("when job does not exists", func() {
					BeforeEach(func() {
						team.JobReturns(atc.Job{}, false, nil)
					})

					It("returns an error", func() {
						_, err := GetBuild(client, team, expectedJobName, "", expectedPipelineName)
						Expect(err).To(MatchError("job not found"))
					})
				})
			})

			Context("when an error was encountered while looking up for team job", func() {
				BeforeEach(func() {
					team.JobReturns(atc.Job{}, false, errors.New("some-error"))
				})

				It("should return an error", func() {
					_, err := GetBuild(client, team, expectedJobName, "", "")
					Expect(err).To(HaveOccurred())
					Expect(err).To(MatchError("failed to get job some-error"))
				})
			})

		})

		Context("when passed pipeline, job, and build names", func() {
			Context("when the build exists", func() {
				BeforeEach(func() {
					team.JobBuildReturns(expectedBuild, true, nil)
				})

				It("returns the build", func() {
					build, err := GetBuild(client, team, expectedJobName, expectedBuildName, expectedPipelineName)
					Expect(err).NotTo(HaveOccurred())
					Expect(build).To(Equal(expectedBuild))
					Expect(team.JobBuildCallCount()).To(Equal(1))
					pipelineName, jobName, buildName := team.JobBuildArgsForCall(0)
					Expect(pipelineName).To(Equal(expectedPipelineName))
					Expect(buildName).To(Equal(expectedBuildName))
					Expect(jobName).To(Equal(expectedJobName))
				})
			})

			Context("when the build does not exist", func() {
				BeforeEach(func() {
					team.JobBuildReturns(atc.Build{}, false, nil)
				})

				It("returns an error", func() {
					_, err := GetBuild(client, team, expectedJobName, expectedBuildName, expectedPipelineName)
					Expect(err).To(MatchError("build not found"))
				})
			})
		})

		Context("when nothing is passed", func() {
			Context("when client.Builds does not return an error", func() {
				var allBuilds [300]atc.Build

				expectedOneOffBuild := atc.Build{
					ID:      150,
					Name:    expectedBuildName,
					Status:  "success",
					JobName: "",
					APIURL:  fmt.Sprintf("api/v1/builds/%s", expectedBuildID),
				}

				Context("when a build was found", func() {
					BeforeEach(func() {
						for i := 300 - 1; i >= 0; i-- {
							allBuilds[i] = atc.Build{
								ID:      i,
								Name:    strconv.Itoa(i),
								JobName: "some-job",
								APIURL:  fmt.Sprintf("api/v1/builds/%d", i),
							}
						}

						allBuilds[150] = expectedOneOffBuild

						client.BuildsStub = func(page concourse.Page) ([]atc.Build, concourse.Pagination, error) {
							var builds []atc.Build
							if page.Since != 0 {
								builds = allBuilds[page.Since : page.Since+page.Limit]
							} else {
								builds = allBuilds[0:page.Limit]
							}

							pagination := concourse.Pagination{
								Previous: &concourse.Page{
									Limit: page.Limit,
									Until: builds[0].ID,
								},
								Next: &concourse.Page{
									Limit: page.Limit,
									Since: builds[len(builds)-1].ID,
								},
							}

							return builds, pagination, nil
						}
					})

					It("returns latest one off build", func() {
						build, err := GetBuild(client, nil, "", "", "")
						Expect(err).NotTo(HaveOccurred())
						Expect(build).To(Equal(expectedOneOffBuild))
						Expect(client.BuildsCallCount()).To(Equal(2))
					})
				})

				Context("when no builds were found ", func() {
					BeforeEach(func() {
						client.BuildsReturns([]atc.Build{}, concourse.Pagination{Next: nil}, nil)
					})

					It("returns an error", func() {
						_, err := GetBuild(client, nil, "", "", "")
						Expect(err).To(HaveOccurred())
						Expect(err).To(MatchError("no builds match job"))
					})
				})
			})

			Context("when client.Builds returns an error", func() {
				BeforeEach(func() {
					client.BuildsReturns(nil, concourse.Pagination{}, errors.New("some-error"))
				})

				It("should return an error", func() {
					_, err := GetBuild(client, nil, "", "", "")
					Expect(err).To(HaveOccurred())
					Expect(err).To(MatchError("failed to get builds some-error"))
				})
			})
		})
	})
	Describe("#GetLatestResourceVersions", func() {
		var team *fakes.FakeTeam
		var resourceVersions []atc.ResourceVersion

		resource := flaghelpers.ResourceFlag{
			PipelineName: "mypipeline",
			ResourceName: "myresource",
		}

		BeforeEach(func() {
			team = new(fakes.FakeTeam)
			resourceVersions = []atc.ResourceVersion{
				{
					ID:      1,
					Version: atc.Version{"version": "v1"},
				},
				{
					ID:      2,
					Version: atc.Version{"version": "v1"},
				},
			}
		})

		When("resource versions exist", func() {
			It("returns latest resource version", func() {
				team.ResourceVersionsReturns(resourceVersions, concourse.Pagination{}, true, nil)
				latestResourceVersion, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v1"})
				Expect(err).NotTo(HaveOccurred())
				Expect(latestResourceVersion.Version).To(Equal(atc.Version{"version": "v1"}))
				Expect(latestResourceVersion.ID).To(Equal(1))
			})
		})

		When("call to resource versions returns an error", func() {
			It("returns an error", func() {
				team.ResourceVersionsReturns(nil, concourse.Pagination{}, false, errors.New("fake error"))
				_, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v1"})
				Expect(err).To(MatchError("fake error"))
			})
		})

		When("call to resource versions returns an empty array", func() {
			It("returns an error", func() {
				team.ResourceVersionsReturns([]atc.ResourceVersion{}, concourse.Pagination{}, true, nil)
				_, err := GetLatestResourceVersion(team, resource, atc.Version{"version": "v2"})
				Expect(err).To(MatchError("could not find version matching {\"version\":\"v2\"}"))
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protokube

import (
	"fmt"
	"path/filepath"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/util/pkg/exec"
)

// BuildEtcdManifest creates the pod spec, based on the etcd cluster
func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {

	pod := &v1.Pod{}
	pod.APIVersion = "v1"
	pod.Kind = "Pod"
	pod.Name = c.PodName
	pod.Namespace = "kube-system"
	pod.Labels = map[string]string{"k8s-app": c.PodName}
	pod.Spec.HostNetwork = true

	// dereference our resource requests if they exist
	// cpu
	var cpuRequest resource.Quantity
	if c.CPURequest != nil {
		cpuRequest = *c.CPURequest
	}

	// memory
	var memoryRequest resource.Quantity
	if c.MemoryRequest != nil {
		memoryRequest = *c.MemoryRequest
	}

	{
		container := v1.Container{
			Name:  "etcd-container",
			Image: c.ImageSource,
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    cpuRequest,
					v1.ResourceMemory: memoryRequest,
				},
			},
			Command: exec.WithTee("/usr/local/bin/etcd", []string{}, "/var/log/etcd.log"),
		}
		// build the environment variables for etcd service
		container.Env = buildEtcdEnvironmentOptions(c)

		container.LivenessProbe = &v1.Probe{
			InitialDelaySeconds: 15,
			TimeoutSeconds:      15,
		}
		// ensure we have the correct probe schema
		if c.isTLS() {
			container.LivenessProbe.TCPSocket = &v1.TCPSocketAction{
				Host: "127.0.0.1",
				Port: intstr.FromInt(c.ClientPort),
			}
		} else {
			container.LivenessProbe.HTTPGet = &v1.HTTPGetAction{
				Host:   "127.0.0.1",
				Port:   intstr.FromInt(c.ClientPort),
				Path:   "/health",
				Scheme: v1.URISchemeHTTP,
			}
		}
		container.Ports = append(container.Ports, v1.ContainerPort{
			Name:          "serverport",
			ContainerPort: int32(c.PeerPort),
			HostPort:      int32(c.PeerPort),
		})
		container.Ports = append(container.Ports, v1.ContainerPort{
			Name:          "clientport",
			ContainerPort: int32(c.ClientPort),
			HostPort:      int32(c.ClientPort),
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "varetcdata",
			MountPath: "/var/etcd/" + c.DataDirName,
			ReadOnly:  false,
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "varlogetcd",
			MountPath: "/var/log/etcd.log",
			ReadOnly:  false,
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "hosts",
			MountPath: "/etc/hosts",
			ReadOnly:  true,
		})
		// add the host path mount to the pod spec
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "varetcdata",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: c.VolumeMountPath + "/var/etcd/" + c.DataDirName,
				},
			},
		})
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "varlogetcd",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: c.LogFile,
				},
			},
		})
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "hosts",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: "/etc/hosts",
				},
			},
		})
		// @check if tls is enabled and mount the directory. It might be worth considering
		// if we you use our own directory in /srv i.e /srv/etcd rather than the default /src/kubernetes
		if c.isTLS() {
			for _, dirname := range buildCertificateDirectories(c) {
				normalized := strings.Replace(dirname, "/", "", -1)
				pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
					Name: normalized,
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: dirname,
						},
					},
				})
				container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
					Name:      normalized,
					MountPath: dirname,
					ReadOnly:  true,
				})
			}
		}

		pod.Spec.Containers = append(pod.Spec.Containers, container)
	}

	if c.BackupStore != "" && c.BackupImage != "" {
		backupContainer := buildEtcdBackupManagerContainer(c)
		pod.Spec.Containers = append(pod.Spec.Containers, *backupContainer)
	}

	kubemanifest.MarkPodAsCritical(pod)
	kubemanifest.MarkPodAsClusterCritical(pod)

	return pod
}

// buildEtcdEnvironmentOptions is responsible for building the environment variables for etcd
// @question should we perhaps make this version specific in prep for v3 support?
func buildEtcdEnvironmentOptions(c *EtcdCluster) []v1.EnvVar {
	var options []v1.EnvVar

	// @check if we are using TLS
	scheme := "http"
	if c.isTLS() {
		scheme = "https"
	}

	// add the default setting for masters - http or https
	options = append(options, []v1.EnvVar{
		{Name: "ETCD_NAME", Value: c.Me.Name},
		{Name: "ETCD_DATA_DIR", Value: "/var/etcd/" + c.DataDirName},
		{Name: "ETCD_LISTEN_PEER_URLS", Value: fmt.Sprintf("%s://0.0.0.0:%d", scheme, c.PeerPort)},
		{Name: "ETCD_LISTEN_CLIENT_URLS", Value: fmt.Sprintf("%s://0.0.0.0:%d", scheme, c.ClientPort)},
		{Name: "ETCD_ADVERTISE_CLIENT_URLS", Value: fmt.Sprintf("%s://%s:%d", scheme, c.Me.InternalName, c.ClientPort)},
		{Name: "ETCD_INITIAL_ADVERTISE_PEER_URLS", Value: fmt.Sprintf("%s://%s:%d", scheme, c.Me.InternalName, c.PeerPort)},
		{Name: "ETCD_INITIAL_CLUSTER_STATE", Value: "new"},
		{Name: "ETCD_INITIAL_CLUSTER_TOKEN", Value: c.ClusterToken}}...)

	// add timeout/hearbeat settings
	if notEmpty(c.ElectionTimeout) {
		options = append(options, v1.EnvVar{Name: "ETCD_ELECTION_TIMEOUT", Value: c.ElectionTimeout})
	}
	if notEmpty(c.HeartbeatInterval) {
		options = append(options, v1.EnvVar{Name: "ETCD_HEARTBEAT_INTERVAL", Value: c.HeartbeatInterval})
	}

	// @check if we are using peer certificates
	if notEmpty(c.PeerCA) {
		options = append(options, []v1.EnvVar{
			{Name: "ETCD_PEER_TRUSTED_CA_FILE", Value: c.PeerCA}}...)
	}
	if notEmpty(c.PeerCert) {
		options = append(options, v1.EnvVar{Name: "ETCD_PEER_CERT_FILE", Value: c.PeerCert})
	}
	if notEmpty(c.PeerKey) {
		options = append(options, v1.EnvVar{Name: "ETCD_PEER_KEY_FILE", Value: c.PeerKey})
	}
	if notEmpty(c.TLSCA) {
		options = append(options, v1.EnvVar{Name: "ETCD_TRUSTED_CA_FILE", Value: c.TLSCA})
	}
	if notEmpty(c.TLSCert) {
		options = append(options, v1.EnvVar{Name: "ETCD_CERT_FILE", Value: c.TLSCert})
	}
	if notEmpty(c.TLSKey) {
		options = append(options, v1.EnvVar{Name: "ETCD_KEY_FILE", Value: c.TLSKey})
	}
	if c.isTLS() {
		if c.TLSAuth {
			options = append(options, v1.EnvVar{Name: "ETCD_CLIENT_CERT_AUTH", Value: "true"})
			options = append(options, v1.EnvVar{Name: "ETCD_PEER_CLIENT_CERT_AUTH", Value: "true"})
		}
	}

	// @step: generate the initial cluster
	var hosts []string
	for _, node := range c.Nodes {
		hosts = append(hosts, node.Name+"="+fmt.Sprintf("%s://%s:%d", scheme, node.InternalName, c.PeerPort))
	}
	options = append(options, v1.EnvVar{Name: "ETCD_INITIAL_CLUSTER", Value: strings.Join(hosts, ",")})

	return options
}

// buildCertificateDirectories generates a list of the base directories which the certificates are located
// so we can map in as volumes. They will probably all be placed into /src/kubernetes, but just to make it
// generic.
func buildCertificateDirectories(c *EtcdCluster) []string {
	tracked := make(map[string]bool)

	for _, x := range []string{c.TLSCA, c.TLSCert, c.TLSKey, c.PeerCA, c.PeerKey, c.PeerKey} {
		if x == "" || tracked[filepath.Dir(x)] {
			continue
		}
		tracked[filepath.Dir(x)] = true
	}

	var list []string
	for k := range tracked {
		list = append(list, k)
	}

	return list
}

// notEmpty is just a code pretty version if string != ""
func notEmpty(v string) bool {
	return v != ""
}

// buildEtcdBackupManagerContainer builds a container for the standalone etcd backup manager
func buildEtcdBackupManagerContainer(c *EtcdCluster) *v1.Container {
	command := []string{"/etcd-backup"}
	command = append(command, "--backup-store", c.BackupStore)
	command = append(command, "--cluster-name", c.ClusterName)
	command = append(command, "--data-dir", "/var/etcd/"+c.DataDirName)

	if c.isTLS() {
		command = append(command, "--client-url", "https://127.0.0.1:4001")
		command = append(command, "--client-ca-file", c.TLSCA)
		command = append(command, "--client-cert-file", c.TLSCert)
		command = append(command, "--client-key-file", c.TLSKey)
	}

	container := v1.Container{
		Name:    "etcd-backup",
		Image:   c.BackupImage,
		Command: command,
	}

	// TODO: TLS options
	// TODO: Liveness probe?

	// volume should already have been registered
	container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
		Name:      "varetcdata",
		MountPath: "/var/etcd/" + c.DataDirName,
		ReadOnly:  false,
	})

	if c.isTLS() {
		for _, dirname := range buildCertificateDirectories(c) {
			normalized := strings.Replace(dirname, "/", "", -1)

			// pod volume already registered for etcd container above
			container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
				Name:      normalized,
				MountPath: dirname,
				ReadOnly:  true,
			})
		}
	}

	return &container
}
<commit_msg>Fix cert dirs in the etcd manifest for Windows<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package protokube

import (
	"fmt"
	"path/filepath"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/util/pkg/exec"
)

// BuildEtcdManifest creates the pod spec, based on the etcd cluster
func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {

	pod := &v1.Pod{}
	pod.APIVersion = "v1"
	pod.Kind = "Pod"
	pod.Name = c.PodName
	pod.Namespace = "kube-system"
	pod.Labels = map[string]string{"k8s-app": c.PodName}
	pod.Spec.HostNetwork = true

	// dereference our resource requests if they exist
	// cpu
	var cpuRequest resource.Quantity
	if c.CPURequest != nil {
		cpuRequest = *c.CPURequest
	}

	// memory
	var memoryRequest resource.Quantity
	if c.MemoryRequest != nil {
		memoryRequest = *c.MemoryRequest
	}

	{
		container := v1.Container{
			Name:  "etcd-container",
			Image: c.ImageSource,
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    cpuRequest,
					v1.ResourceMemory: memoryRequest,
				},
			},
			Command: exec.WithTee("/usr/local/bin/etcd", []string{}, "/var/log/etcd.log"),
		}
		// build the environment variables for etcd service
		container.Env = buildEtcdEnvironmentOptions(c)

		container.LivenessProbe = &v1.Probe{
			InitialDelaySeconds: 15,
			TimeoutSeconds:      15,
		}
		// ensure we have the correct probe schema
		if c.isTLS() {
			container.LivenessProbe.TCPSocket = &v1.TCPSocketAction{
				Host: "127.0.0.1",
				Port: intstr.FromInt(c.ClientPort),
			}
		} else {
			container.LivenessProbe.HTTPGet = &v1.HTTPGetAction{
				Host:   "127.0.0.1",
				Port:   intstr.FromInt(c.ClientPort),
				Path:   "/health",
				Scheme: v1.URISchemeHTTP,
			}
		}
		container.Ports = append(container.Ports, v1.ContainerPort{
			Name:          "serverport",
			ContainerPort: int32(c.PeerPort),
			HostPort:      int32(c.PeerPort),
		})
		container.Ports = append(container.Ports, v1.ContainerPort{
			Name:          "clientport",
			ContainerPort: int32(c.ClientPort),
			HostPort:      int32(c.ClientPort),
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "varetcdata",
			MountPath: "/var/etcd/" + c.DataDirName,
			ReadOnly:  false,
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "varlogetcd",
			MountPath: "/var/log/etcd.log",
			ReadOnly:  false,
		})
		container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
			Name:      "hosts",
			MountPath: "/etc/hosts",
			ReadOnly:  true,
		})
		// add the host path mount to the pod spec
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "varetcdata",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: c.VolumeMountPath + "/var/etcd/" + c.DataDirName,
				},
			},
		})
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "varlogetcd",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: c.LogFile,
				},
			},
		})
		pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
			Name: "hosts",
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: "/etc/hosts",
				},
			},
		})
		// @check if tls is enabled and mount the directory. It might be worth considering
		// if we you use our own directory in /srv i.e /srv/etcd rather than the default /src/kubernetes
		if c.isTLS() {
			for _, dirname := range buildCertificateDirectories(c) {
				normalized := strings.Replace(dirname, "/", "", -1)
				pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{
					Name: normalized,
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: dirname,
						},
					},
				})
				container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
					Name:      normalized,
					MountPath: dirname,
					ReadOnly:  true,
				})
			}
		}

		pod.Spec.Containers = append(pod.Spec.Containers, container)
	}

	if c.BackupStore != "" && c.BackupImage != "" {
		backupContainer := buildEtcdBackupManagerContainer(c)
		pod.Spec.Containers = append(pod.Spec.Containers, *backupContainer)
	}

	kubemanifest.MarkPodAsCritical(pod)
	kubemanifest.MarkPodAsClusterCritical(pod)

	return pod
}

// buildEtcdEnvironmentOptions is responsible for building the environment variables for etcd
// @question should we perhaps make this version specific in prep for v3 support?
func buildEtcdEnvironmentOptions(c *EtcdCluster) []v1.EnvVar {
	var options []v1.EnvVar

	// @check if we are using TLS
	scheme := "http"
	if c.isTLS() {
		scheme = "https"
	}

	// add the default setting for masters - http or https
	options = append(options, []v1.EnvVar{
		{Name: "ETCD_NAME", Value: c.Me.Name},
		{Name: "ETCD_DATA_DIR", Value: "/var/etcd/" + c.DataDirName},
		{Name: "ETCD_LISTEN_PEER_URLS", Value: fmt.Sprintf("%s://0.0.0.0:%d", scheme, c.PeerPort)},
		{Name: "ETCD_LISTEN_CLIENT_URLS", Value: fmt.Sprintf("%s://0.0.0.0:%d", scheme, c.ClientPort)},
		{Name: "ETCD_ADVERTISE_CLIENT_URLS", Value: fmt.Sprintf("%s://%s:%d", scheme, c.Me.InternalName, c.ClientPort)},
		{Name: "ETCD_INITIAL_ADVERTISE_PEER_URLS", Value: fmt.Sprintf("%s://%s:%d", scheme, c.Me.InternalName, c.PeerPort)},
		{Name: "ETCD_INITIAL_CLUSTER_STATE", Value: "new"},
		{Name: "ETCD_INITIAL_CLUSTER_TOKEN", Value: c.ClusterToken}}...)

	// add timeout/hearbeat settings
	if notEmpty(c.ElectionTimeout) {
		options = append(options, v1.EnvVar{Name: "ETCD_ELECTION_TIMEOUT", Value: c.ElectionTimeout})
	}
	if notEmpty(c.HeartbeatInterval) {
		options = append(options, v1.EnvVar{Name: "ETCD_HEARTBEAT_INTERVAL", Value: c.HeartbeatInterval})
	}

	// @check if we are using peer certificates
	if notEmpty(c.PeerCA) {
		options = append(options, []v1.EnvVar{
			{Name: "ETCD_PEER_TRUSTED_CA_FILE", Value: c.PeerCA}}...)
	}
	if notEmpty(c.PeerCert) {
		options = append(options, v1.EnvVar{Name: "ETCD_PEER_CERT_FILE", Value: c.PeerCert})
	}
	if notEmpty(c.PeerKey) {
		options = append(options, v1.EnvVar{Name: "ETCD_PEER_KEY_FILE", Value: c.PeerKey})
	}
	if notEmpty(c.TLSCA) {
		options = append(options, v1.EnvVar{Name: "ETCD_TRUSTED_CA_FILE", Value: c.TLSCA})
	}
	if notEmpty(c.TLSCert) {
		options = append(options, v1.EnvVar{Name: "ETCD_CERT_FILE", Value: c.TLSCert})
	}
	if notEmpty(c.TLSKey) {
		options = append(options, v1.EnvVar{Name: "ETCD_KEY_FILE", Value: c.TLSKey})
	}
	if c.isTLS() {
		if c.TLSAuth {
			options = append(options, v1.EnvVar{Name: "ETCD_CLIENT_CERT_AUTH", Value: "true"})
			options = append(options, v1.EnvVar{Name: "ETCD_PEER_CLIENT_CERT_AUTH", Value: "true"})
		}
	}

	// @step: generate the initial cluster
	var hosts []string
	for _, node := range c.Nodes {
		hosts = append(hosts, node.Name+"="+fmt.Sprintf("%s://%s:%d", scheme, node.InternalName, c.PeerPort))
	}
	options = append(options, v1.EnvVar{Name: "ETCD_INITIAL_CLUSTER", Value: strings.Join(hosts, ",")})

	return options
}

// buildCertificateDirectories generates a list of the base directories which the certificates are located
// so we can map in as volumes. They will probably all be placed into /src/kubernetes, but just to make it
// generic.
func buildCertificateDirectories(c *EtcdCluster) []string {
	tracked := make(map[string]bool)

	for _, x := range []string{c.TLSCA, c.TLSCert, c.TLSKey, c.PeerCA, c.PeerKey, c.PeerKey} {
		dir := filepath.ToSlash(filepath.Dir(x))
		if x == "" || tracked[dir] {
			continue
		}
		tracked[dir] = true
	}

	var list []string
	for k := range tracked {
		list = append(list, k)
	}

	return list
}

// notEmpty is just a code pretty version if string != ""
func notEmpty(v string) bool {
	return v != ""
}

// buildEtcdBackupManagerContainer builds a container for the standalone etcd backup manager
func buildEtcdBackupManagerContainer(c *EtcdCluster) *v1.Container {
	command := []string{"/etcd-backup"}
	command = append(command, "--backup-store", c.BackupStore)
	command = append(command, "--cluster-name", c.ClusterName)
	command = append(command, "--data-dir", "/var/etcd/"+c.DataDirName)

	if c.isTLS() {
		command = append(command, "--client-url", "https://127.0.0.1:4001")
		command = append(command, "--client-ca-file", c.TLSCA)
		command = append(command, "--client-cert-file", c.TLSCert)
		command = append(command, "--client-key-file", c.TLSKey)
	}

	container := v1.Container{
		Name:    "etcd-backup",
		Image:   c.BackupImage,
		Command: command,
	}

	// TODO: TLS options
	// TODO: Liveness probe?

	// volume should already have been registered
	container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
		Name:      "varetcdata",
		MountPath: "/var/etcd/" + c.DataDirName,
		ReadOnly:  false,
	})

	if c.isTLS() {
		for _, dirname := range buildCertificateDirectories(c) {
			normalized := strings.Replace(dirname, "/", "", -1)

			// pod volume already registered for etcd container above
			container.VolumeMounts = append(container.VolumeMounts, v1.VolumeMount{
				Name:      normalized,
				MountPath: dirname,
				ReadOnly:  true,
			})
		}
	}

	return &container
}
<|endoftext|>


<commit_before>package provider

import (
	"github.com/hashicorp/terraform/helper/schema"
	"net/http"
	"fmt"
	"strconv"
	"os"

	sumo "github.com/erikvanbrakel/terraform-provider-sumologic/go-sumologic"
)

func resourceSumologicCollector() *schema.Resource {
	return &schema.Resource {
		Create: resourceSumologicCollectorCreate,
		Read: resourceSumologicCollectorRead,
		Delete: resourceSumologicCollectorDelete,

		Schema: map[string]*schema.Schema {
			"name" : {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"category" : {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceSumologicCollectorRead(d *schema.ResourceData, meta interface{}) error {

	c := meta.(*sumo.SumologicClient)

	id, err := strconv.Atoi(d.Id())

	if err != nil {
		return err
	}

	collector, err := c.GetCollector(id)

	if err != nil {
		return err
	}

	d.Set("name", collector.Name)
	d.Set("description", collector.Description)
	d.Set("category", collector.Category)

	return nil
}

func resourceSumologicCollectorDelete(d *schema.ResourceData, meta interface{}) error {

	accessId := os.Getenv("SL_ACCESSID")
	accessKey := os.Getenv("SL_ACCESSKEY")
	url := fmt.Sprintf("https://%s:%s@api.eu.sumologic.com/api/v1/collectors/%s", accessId, accessKey, d.Id())

	req, err := http.NewRequest(http.MethodDelete, url, nil)

	if err != nil {
		return err
	}
	_, err = http.DefaultClient.Do(req)

	if err != nil {
		return err
	}

	return nil
}

func resourceSumologicCollectorCreate(d *schema.ResourceData, meta interface{}) error {
	c := meta.(*sumo.SumologicClient)

	collector, err := c.CreateCollector(
		"Hosted",
		d.Get("name").(string),
		d.Get("description").(string),
		d.Get("category").(string),
	)

	if err != nil {
		return err
	}

	d.SetId(strconv.Itoa(collector.Id))

	return resourceSumologicCollectorRead(d, meta)
}
<commit_msg>Fix: Delete method didn't use client yet<commit_after>package provider

import (
	"github.com/hashicorp/terraform/helper/schema"
	"strconv"

	sumo "github.com/erikvanbrakel/terraform-provider-sumologic/go-sumologic"
)

func resourceSumologicCollector() *schema.Resource {
	return &schema.Resource {
		Create: resourceSumologicCollectorCreate,
		Read: resourceSumologicCollectorRead,
		Delete: resourceSumologicCollectorDelete,

		Schema: map[string]*schema.Schema {
			"name" : {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"description": {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"category" : {
				Type: schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceSumologicCollectorRead(d *schema.ResourceData, meta interface{}) error {

	c := meta.(*sumo.SumologicClient)

	id, err := strconv.Atoi(d.Id())

	if err != nil {
		return err
	}

	collector, err := c.GetCollector(id)

	if err != nil {
		return err
	}

	d.Set("name", collector.Name)
	d.Set("description", collector.Description)
	d.Set("category", collector.Category)

	return nil
}

func resourceSumologicCollectorDelete(d *schema.ResourceData, meta interface{}) error {

	c := meta.(*sumo.SumologicClient)

	id, _ := strconv.Atoi(d.Id())
	return c.DeleteCollector(id)
}

func resourceSumologicCollectorCreate(d *schema.ResourceData, meta interface{}) error {
	c := meta.(*sumo.SumologicClient)

	collector, err := c.CreateCollector(
		"Hosted",
		d.Get("name").(string),
		d.Get("description").(string),
		d.Get("category").(string),
	)

	if err != nil {
		return err
	}

	d.SetId(strconv.Itoa(collector.Id))

	return resourceSumologicCollectorRead(d, meta)
}
<|endoftext|>