<commit_before>package processor

import (
	"bytes"
	"fmt"
	"github.com/Pixboost/transformimgs/v6/img"
	"github.com/Pixboost/transformimgs/v6/img/processor/internal"
	"os/exec"
	"strconv"
)

type ImageMagick struct {
	convertCmd  string
	identifyCmd string
	// AdditionalArgs are static arguments that will be passed to ImageMagick "convert" command for all operations.
	// Argument name and value should be in separate array elements.
	AdditionalArgs []string
	// GetAdditionalArgs could return additional argument to ImageMagick "convert" command.
	// "op" is the name of the operation: "optimise", "resize" or "fit".
	// Argument name and value should be in separate array elements.
	GetAdditionalArgs func(op string, image []byte, imageInfo *img.Info) []string
}

var convertOpts = []string{
	"-unsharp", "0.25x0.08+8.3+0.045",
	"-dither", "None",
	"-posterize", "136",
	"-define", "jpeg:fancy-upsampling=off",
	"-define", "png:compression-filter=5",
	"-define", "png:compression-level=9",
	"-define", "png:compression-strategy=0",
	"-define", "png:exclude-chunk=bKGD,cHRM,EXIF,gAMA,iCCP,iTXt,sRGB,tEXt,zCCP,zTXt,date",
	"-define", "webp:method=6",
	"-interlace", "None",
	"-colorspace", "sRGB",
	"-sampling-factor", "4:2:0",
	"+profile", "!icc,*",
}

var cutToFitOpts = []string{
	"-gravity", "center",
}

//If set then will print all commands to stdout.
var Debug bool = true

const (
	MaxWebpWidth  = 16383
	MaxWebpHeight = 16383
)

// NewImageMagick creates a new ImageMagick processor. It does require
// ImageMagick binaries to be installed on the local machine.
//
// im is a path to ImageMagick "convert" binary.
// idi is a path to ImageMagick "identify" binary.
func NewImageMagick(im string, idi string) (*ImageMagick, error) {
	if len(im) == 0 {
		img.Log.Error("Path to \"convert\" command should be set by -imConvert flag")
		return nil, fmt.Errorf("path to imagemagick convert binary must be provided")
	}
	if len(idi) == 0 {
		img.Log.Error("Path to \"identify\" command should be set by -imIdentify flag")
		return nil, fmt.Errorf("path to imagemagick identify binary must be provided")
	}

	_, err := exec.LookPath(im)
	if err != nil {
		return nil, err
	}
	_, err = exec.LookPath(idi)
	if err != nil {
		return nil, err
	}

	return &ImageMagick{
		convertCmd:     im,
		identifyCmd:    idi,
		AdditionalArgs: []string{},
	}, nil
}

// Resize resizes an image to the given size preserving aspect ratio. No cropping applies.
//
// Format of the size argument is WIDTHxHEIGHT with any of the dimension could be dropped, e.g. 300, x200, 300x200.
func (p *ImageMagick) Resize(data []byte, size string, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
	}
	err = internal.CalculateTargetSizeForResize(source, target, size)
	if err != nil {
		img.Log.Errorf("could not calculate target size for [%s], size: [%s]\n", imgId, size)
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size)
	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("resize", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	outputImageData, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	return &img.Image{
		Data:     outputImageData,
		MimeType: mimeType,
	}, nil
}

// FitToSize resizes input image to exact size with cropping everything that out of the bound.
// It doesn't respect the aspect ratio of the original image.
//
// Format of the size argument is WIDTHxHEIGHT, e.g. 300x200. Both dimensions must be included.
func (p *ImageMagick) FitToSize(data []byte, size string, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
	}
	err = internal.CalculateTargetSizeForFit(target, size)
	if err != nil {
		img.Log.Errorf("could not calculate target size for [%s], size: [%s]\n", imgId, size)
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size+"^")

	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("fit", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, cutToFitOpts...)
	args = append(args, "-extent", size)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	outputImageData, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	return &img.Image{
		Data:     outputImageData,
		MimeType: mimeType,
	}, nil
}

func (p *ImageMagick) Optimise(data []byte, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
		Width:  source.Width,
		Height: source.Height,
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input

	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("optimise", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	result, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	if len(result) > len(data) {
		img.Log.Printf("[%s] WARNING: Optimised size [%d] is more than original [%d], fallback to original", imgId, len(result), len(data))
		result = data
	}

	return &img.Image{
		Data:     result,
		MimeType: mimeType,
	}, nil
}

func (p *ImageMagick) execImagemagick(in *bytes.Reader, args []string, imgId string) ([]byte, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.convertCmd)

	cmd.Args = append(cmd.Args, args...)

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	if Debug {
		img.Log.Printf("[%s] Running resize command, args '%v'\n", imgId, cmd.Args)
	}
	err := cmd.Run()
	if err != nil {
		img.Log.Printf("[%s] Error executing convert command: %s\n", imgId, err.Error())
		img.Log.Printf("[%s] ERROR: %s\n", imgId, cmderr.String())
		return nil, err
	}

	return out.Bytes(), nil
}

func (p *ImageMagick) loadImageInfo(in *bytes.Reader, imgId string) (*img.Info, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.identifyCmd)
	cmd.Args = append(cmd.Args, "-format", "%m %Q %[opaque] %w %h", "-")

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	if Debug {
		img.Log.Printf("[%s] Running identify command, args '%v'\n", imgId, cmd.Args)
	}
	err := cmd.Run()
	if err != nil {
		img.Log.Printf("[%s] Error executing identify command: %s\n", err.Error(), imgId)
		img.Log.Printf("[%s] ERROR: %s\n", cmderr.String(), imgId)
		return nil, err
	}

	imageInfo := &img.Info{}
	_, err = fmt.Sscanf(out.String(), "%s %d %t %d %d", &imageInfo.Format, &imageInfo.Quality, &imageInfo.Opaque, &imageInfo.Width, &imageInfo.Height)
	if err != nil {
		return nil, err
	}

	return imageInfo, nil
}

func getOutputFormat(src *img.Info, target *img.Info, supportedFormats []string) (string, string) {
	webP := false
	avif := false
	for _, f := range supportedFormats {
		if f == "image/webp" && src.Height < MaxWebpHeight && src.Width < MaxWebpWidth {
			webP = true
		}
		// ImageMagick doesn't support encoding of alpha channel for AVIF.
		// Converting 1000x1000 image into AVIF will consume about 500Mb of RAM.
		targetSize := target.Width * target.Height
		if f == "image/avif" && src.Opaque && targetSize < (1000*1000) && targetSize != 0 {
			avif = true
		}
	}

	if avif {
		return "avif:-", "image/avif"
	}
	if webP {
		return "webp:-", "image/webp"
	}

	return "-", ""
}

func getConvertFormatOptions(inf *img.Info) []string {
	if inf.Format == "PNG" {
		opts := []string{
			"-define", "webp:lossless=true",
		}
		if inf.Opaque {
			opts = append(opts, "-colors", "256")
		}
		return opts
	}

	return []string{}
}

func getQualityOptions(source *img.Info, target *img.Info, outputMimeType string) []string {
	if source.Quality == 100 {
		return []string{"-quality", "82"}
	}

	if outputMimeType == "image/avif" {
		var addQuality = 0
		if (target.Height * target.Width) > (500 * 500) {
			addQuality = 10
		}

		if source.Quality > 85 {
			return []string{"-quality", strconv.Itoa(70 + addQuality)}
		} else if source.Quality > 75 {
			return []string{"-quality", strconv.Itoa(60 + addQuality)}
		} else {
			return []string{"-quality", strconv.Itoa(50 + addQuality)}
		}
	}

	return []string{}
}
<commit_msg>Removed quality constraints for AVIF<commit_after>package processor

import (
	"bytes"
	"fmt"
	"github.com/Pixboost/transformimgs/v6/img"
	"github.com/Pixboost/transformimgs/v6/img/processor/internal"
	"os/exec"
)

type ImageMagick struct {
	convertCmd  string
	identifyCmd string
	// AdditionalArgs are static arguments that will be passed to ImageMagick "convert" command for all operations.
	// Argument name and value should be in separate array elements.
	AdditionalArgs []string
	// GetAdditionalArgs could return additional argument to ImageMagick "convert" command.
	// "op" is the name of the operation: "optimise", "resize" or "fit".
	// Argument name and value should be in separate array elements.
	GetAdditionalArgs func(op string, image []byte, imageInfo *img.Info) []string
}

var convertOpts = []string{
	"-unsharp", "0.25x0.08+8.3+0.045",
	"-dither", "None",
	"-posterize", "136",
	"-define", "jpeg:fancy-upsampling=off",
	"-define", "png:compression-filter=5",
	"-define", "png:compression-level=9",
	"-define", "png:compression-strategy=0",
	"-define", "png:exclude-chunk=bKGD,cHRM,EXIF,gAMA,iCCP,iTXt,sRGB,tEXt,zCCP,zTXt,date",
	"-define", "webp:method=6",
	"-interlace", "None",
	"-colorspace", "sRGB",
	"-sampling-factor", "4:2:0",
	"+profile", "!icc,*",
}

var cutToFitOpts = []string{
	"-gravity", "center",
}

//If set then will print all commands to stdout.
var Debug bool = true

const (
	MaxWebpWidth  = 16383
	MaxWebpHeight = 16383

	// There are two aspects to this:
	// * Encoding to AVIF consumes a lot of memory
	// * On big sizes quality of Webp is better (could be a codec thing rather than a format)
	MaxAVIFTargetSize = 1000 * 1000
)

// NewImageMagick creates a new ImageMagick processor. It does require
// ImageMagick binaries to be installed on the local machine.
//
// im is a path to ImageMagick "convert" binary.
// idi is a path to ImageMagick "identify" binary.
func NewImageMagick(im string, idi string) (*ImageMagick, error) {
	if len(im) == 0 {
		img.Log.Error("Path to \"convert\" command should be set by -imConvert flag")
		return nil, fmt.Errorf("path to imagemagick convert binary must be provided")
	}
	if len(idi) == 0 {
		img.Log.Error("Path to \"identify\" command should be set by -imIdentify flag")
		return nil, fmt.Errorf("path to imagemagick identify binary must be provided")
	}

	_, err := exec.LookPath(im)
	if err != nil {
		return nil, err
	}
	_, err = exec.LookPath(idi)
	if err != nil {
		return nil, err
	}

	return &ImageMagick{
		convertCmd:     im,
		identifyCmd:    idi,
		AdditionalArgs: []string{},
	}, nil
}

// Resize resizes an image to the given size preserving aspect ratio. No cropping applies.
//
// Format of the size argument is WIDTHxHEIGHT with any of the dimension could be dropped, e.g. 300, x200, 300x200.
func (p *ImageMagick) Resize(data []byte, size string, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
	}
	err = internal.CalculateTargetSizeForResize(source, target, size)
	if err != nil {
		img.Log.Errorf("could not calculate target size for [%s], size: [%s]\n", imgId, size)
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size)
	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("resize", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	outputImageData, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	return &img.Image{
		Data:     outputImageData,
		MimeType: mimeType,
	}, nil
}

// FitToSize resizes input image to exact size with cropping everything that out of the bound.
// It doesn't respect the aspect ratio of the original image.
//
// Format of the size argument is WIDTHxHEIGHT, e.g. 300x200. Both dimensions must be included.
func (p *ImageMagick) FitToSize(data []byte, size string, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
	}
	err = internal.CalculateTargetSizeForFit(target, size)
	if err != nil {
		img.Log.Errorf("could not calculate target size for [%s], size: [%s]\n", imgId, size)
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input
	args = append(args, "-resize", size+"^")

	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("fit", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, cutToFitOpts...)
	args = append(args, "-extent", size)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	outputImageData, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	return &img.Image{
		Data:     outputImageData,
		MimeType: mimeType,
	}, nil
}

func (p *ImageMagick) Optimise(data []byte, imgId string, supportedFormats []string) (*img.Image, error) {
	source, err := p.loadImageInfo(bytes.NewReader(data), imgId)
	if err != nil {
		return nil, err
	}

	target := &img.Info{
		Opaque: source.Opaque,
		Width:  source.Width,
		Height: source.Height,
	}
	outputFormatArg, mimeType := getOutputFormat(source, target, supportedFormats)

	args := make([]string, 0)
	args = append(args, "-") //Input

	args = append(args, getQualityOptions(source, target, mimeType)...)
	args = append(args, p.AdditionalArgs...)
	if p.GetAdditionalArgs != nil {
		args = append(args, p.GetAdditionalArgs("optimise", data, source)...)
	}
	args = append(args, convertOpts...)
	args = append(args, getConvertFormatOptions(source)...)
	args = append(args, outputFormatArg) //Output

	result, err := p.execImagemagick(bytes.NewReader(data), args, imgId)
	if err != nil {
		return nil, err
	}

	if len(result) > len(data) {
		img.Log.Printf("[%s] WARNING: Optimised size [%d] is more than original [%d], fallback to original", imgId, len(result), len(data))
		result = data
	}

	return &img.Image{
		Data:     result,
		MimeType: mimeType,
	}, nil
}

func (p *ImageMagick) execImagemagick(in *bytes.Reader, args []string, imgId string) ([]byte, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.convertCmd)

	cmd.Args = append(cmd.Args, args...)

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	if Debug {
		img.Log.Printf("[%s] Running resize command, args '%v'\n", imgId, cmd.Args)
	}
	err := cmd.Run()
	if err != nil {
		img.Log.Printf("[%s] Error executing convert command: %s\n", imgId, err.Error())
		img.Log.Printf("[%s] ERROR: %s\n", imgId, cmderr.String())
		return nil, err
	}

	return out.Bytes(), nil
}

func (p *ImageMagick) loadImageInfo(in *bytes.Reader, imgId string) (*img.Info, error) {
	var out, cmderr bytes.Buffer
	cmd := exec.Command(p.identifyCmd)
	cmd.Args = append(cmd.Args, "-format", "%m %Q %[opaque] %w %h", "-")

	cmd.Stdin = in
	cmd.Stdout = &out
	cmd.Stderr = &cmderr

	if Debug {
		img.Log.Printf("[%s] Running identify command, args '%v'\n", imgId, cmd.Args)
	}
	err := cmd.Run()
	if err != nil {
		img.Log.Printf("[%s] Error executing identify command: %s\n", err.Error(), imgId)
		img.Log.Printf("[%s] ERROR: %s\n", cmderr.String(), imgId)
		return nil, err
	}

	imageInfo := &img.Info{}
	_, err = fmt.Sscanf(out.String(), "%s %d %t %d %d", &imageInfo.Format, &imageInfo.Quality, &imageInfo.Opaque, &imageInfo.Width, &imageInfo.Height)
	if err != nil {
		return nil, err
	}

	return imageInfo, nil
}

func getOutputFormat(src *img.Info, target *img.Info, supportedFormats []string) (string, string) {
	webP := false
	avif := false
	for _, f := range supportedFormats {
		if f == "image/webp" && src.Height < MaxWebpHeight && src.Width < MaxWebpWidth {
			webP = true
		}
		targetSize := target.Width * target.Height

		// ImageMagick doesn't support encoding of alpha channel for AVIF.
		if f == "image/avif" && src.Opaque && targetSize < MaxAVIFTargetSize && targetSize != 0 {
			avif = true
		}
	}

	if avif {
		return "avif:-", "image/avif"
	}
	if webP {
		return "webp:-", "image/webp"
	}

	return "-", ""
}

func getConvertFormatOptions(inf *img.Info) []string {
	if inf.Format == "PNG" {
		opts := []string{
			"-define", "webp:lossless=true",
		}
		if inf.Opaque {
			opts = append(opts, "-colors", "256")
		}
		return opts
	}

	return []string{}
}

func getQualityOptions(source *img.Info, target *img.Info, outputMimeType string) []string {
	if source.Quality == 100 {
		return []string{"-quality", "82"}
	}

	if outputMimeType == "image/avif" {
		if source.Quality > 85 {
			return []string{"-quality", "70"}
		} else if source.Quality > 75 {
			return []string{"-quality", "60"}
		} else {
			return []string{"-quality", "50"}
		}
	}

	return []string{}
}
<|endoftext|>


<commit_before>package api

import (
	"log"
	"net/http"
	"time"

	//"github.com/MyHomeworkSpace/api-server/auth"

	"github.com/labstack/echo"
)

// structs for data
type PlannerAnnouncement struct {
	ID int `json:"id"`
	Date string `json:"date"`
	Text string `json:"text"`
}

type PlannerEvent struct {
	EventID int `json:"eventId"`
	SectionIndex int `json:"sectionIndex"`
	UserID int `json:"userId"`
	SubID int `json:"subId"`
	Date string `json:"date"`
	Text string `json:"text"`
	Done int `json:"done"`
}

type PlannerFriday struct {
	EntryID int `json:"entryId"`
	Date string `json:"date"`
	Index int `json:"index"`
}

type PlannerSection struct {
	SectionGID int `json:"sectionGid"`
	SectionIndex int `json:"sectionIndex"`
	UserID int `json:"userId"`
	Name string `json:"name"`
}

// responses
type PlannerAnnouncementResponse struct {
	Status string `json:"status"`
	Announcement PlannerAnnouncement `json:"announcement"`
}

type MultiPlannerAnnouncementResponse struct {
	Status string `json:"status"`
	Announcements []PlannerAnnouncement `json:"announcements"`
}

type PlannerFridayResponse struct {
	Status string `json:"status"`
	Friday PlannerFriday `json:"friday"`
}

type MultiPlannerSectionResponse struct {
	Status string `json:"status"`
	Sections []PlannerSection `json:"sections"`
}

type PlannerWeekResponse struct {
	Status string `json:"status"`
	StartDate string `json:"startDate"`
	Announcements []PlannerAnnouncement `json:"announcements"`
	Events []PlannerEvent `json:"events"`
}

func InitPlannerAPI(e *echo.Echo) {
	e.GET("/planner/announcements/get/:date", func(c echo.Context) error {
		rows, err := DB.Query("SELECT id, date, text FROM planner_announcements WHERE date = ?", c.Param("date"))
		if err != nil {
			log.Println("Error while getting announcement information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		if rows.Next() {
			resp := PlannerAnnouncement{-1, "", ""}
			rows.Scan(&resp.ID, &resp.Date, &resp.Text)
			jsonResp := PlannerAnnouncementResponse{"ok", resp}
			return c.JSON(http.StatusOK, jsonResp)
		} else {
			jsonResp := StatusResponse{"ok"}
			return c.JSON(http.StatusOK, jsonResp)
		}
	})

	e.GET("/planner/announcements/getWeek/:date", func(c echo.Context) error {
		startDate, err := time.Parse("2006-01-02", c.Param("date"))
		if err != nil {
			jsonResp := ErrorResponse{"error", "Invalid date."}
			return c.JSON(http.StatusBadRequest, jsonResp)
		}
		endDate := startDate.Add(time.Hour * 24 * 7)
		rows, err := DB.Query("SELECT id, date, text FROM planner_announcements WHERE date >= ? AND date <= ?", startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
		if err != nil {
			log.Println("Error while getting announcement information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		announcements := []PlannerAnnouncement{}
		for rows.Next() {
			resp := PlannerAnnouncement{-1, "", ""}
			rows.Scan(&resp.ID, &resp.Date, &resp.Text)
			announcements = append(announcements, resp)
		}
		jsonResp := MultiPlannerAnnouncementResponse{"ok", announcements}
		return c.JSON(http.StatusOK, jsonResp)
	})

	e.GET("/planner/fridays/get/:date", func(c echo.Context) error {
		rows, err := DB.Query("SELECT * FROM planner_fridays WHERE date = ?", c.Param("date"))
		if err != nil {
			log.Println("Error while getting friday information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		if rows.Next() {
			resp := PlannerFriday{-1, "", -1}
			rows.Scan(&resp.EntryID, &resp.Date, &resp.Index)
			jsonResp := PlannerFridayResponse{"ok", resp}
			return c.JSON(http.StatusOK, jsonResp)
		} else {
			jsonResp := StatusResponse{"ok"}
			return c.JSON(http.StatusOK, jsonResp)
		}
	})

	// TODO: other /events/get* things? not sure if they're needed really...

	e.GET("/planner/events/getWholeWeek/:date", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			jsonResp := ErrorResponse{"error", "logged_out"}
			return c.JSON(http.StatusUnauthorized, jsonResp)
		}
		startDate, err := time.Parse("2006-01-02", c.Param("date"))
		if err != nil {
			jsonResp := ErrorResponse{"error", "Invalid date."}
			return c.JSON(http.StatusBadRequest, jsonResp)
		}
		endDate := startDate.Add(time.Hour * 24 * 7)

		announcements := []PlannerAnnouncement{}
		events := []PlannerEvent{}

		announcementRows, err := DB.Query("SELECT * FROM planner_announcements WHERE date >= ? AND date <= ?", startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
		if err != nil {
			log.Println("Error while getting week information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer announcementRows.Close()
		for announcementRows.Next() {
			resp := PlannerAnnouncement{-1, "", ""}
			announcementRows.Scan(&resp.ID, &resp.Date, &resp.Text)
			announcements = append(announcements, resp)
		}

		eventRows, err := DB.Query("SELECT * FROM planner_events WHERE userId = ? AND date >= ? AND date <= ?", GetSessionUserID(&c), startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
		if err != nil {
			log.Println("Error while getting week information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer eventRows.Close()
		for eventRows.Next() {
			resp := PlannerEvent{-1, -1, -1, -1, "", "", -1}
			eventRows.Scan(&resp.EventID, &resp.SectionIndex, &resp.UserID, &resp.SubID, &resp.Date, &resp.Text, &resp.Done)
			events = append(events, resp)
		}

		jsonResp := PlannerWeekResponse{"ok", startDate.Format("2006-01-02"), announcements, events}
		return c.JSON(http.StatusOK, jsonResp)
	})

	// /events/post

	// /events/purgeLine

	e.GET("/planner/sections/get", func(c echo.Context) error {
		if GetSessionUserID(&c) == -1 {
			jsonResp := ErrorResponse{"error", "logged_out"}
			return c.JSON(http.StatusUnauthorized, jsonResp)
		}
		rows, err := DB.Query("SELECT sectionGid, sectionIndex, userId, name FROM planner_sections WHERE userId = ?", GetSessionUserID(&c))
		if err != nil {
			log.Println("Error while getting section information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		sections := []PlannerSection{}
		for rows.Next() {
			resp := PlannerSection{-1, -1, -1, ""}
			rows.Scan(&resp.SectionGID, &resp.SectionIndex, &resp.UserID, &resp.Name)
			sections = append(sections, resp)
		}
		jsonResp := MultiPlannerSectionResponse{"ok", sections}
		return c.JSON(http.StatusOK, jsonResp)
	})

	e.POST("/planner/sections/add", func(c echo.Context) error {
		if c.FormValue("name") == "" {
			jsonResp := ErrorResponse{"error", "Missing required parameters."}
			return c.JSON(http.StatusUnprocessableEntity, jsonResp)
		}
		if GetSessionUserID(&c) == -1 {
			jsonResp := ErrorResponse{"error", "logged_out"}
			return c.JSON(http.StatusUnauthorized, jsonResp)
		}
		rows, err := DB.Query("SELECT sectionIndex FROM planner_sections WHERE userId = ? ORDER BY sectionIndex DESC", GetSessionUserID(&c))
		if err != nil {
			log.Println("Error while adding section: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		rows.Next()
		newIndex := -1
		rows.Scan(&newIndex)
		newIndex++

		// add the new section
		stmt, err := DB.Prepare("INSERT INTO planner_sections(sectionIndex, userId, name) VALUES(?, ?, ?)")
		if err != nil {
			log.Println("Error while adding section: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		_, err = stmt.Exec(newIndex, GetSessionUserID(&c), c.FormValue("name"))
		if err != nil {
			log.Println("Error while adding section: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}

		jsonResp := StatusResponse{"ok"}
		return c.JSON(http.StatusOK, jsonResp)
	})
}
<commit_msg>simplify planner API calls<commit_after>package api

import (
	"log"
	"net/http"
	"time"

	//"github.com/MyHomeworkSpace/api-server/auth"

	"github.com/labstack/echo"
)

// structs for data
type PlannerAnnouncement struct {
	ID int `json:"id"`
	Date string `json:"date"`
	Text string `json:"text"`
}

type PlannerFriday struct {
	ID int `json:"id"`
	Date string `json:"date"`
	Index int `json:"index"`
}

// responses
type MultiPlannerAnnouncementResponse struct {
	Status string `json:"status"`
	Announcements []PlannerAnnouncement `json:"announcements"`
}

type PlannerFridayResponse struct {
	Status string `json:"status"`
	Friday PlannerFriday `json:"friday"`
}

func InitPlannerAPI(e *echo.Echo) {
	e.GET("/planner/announcements/getWeek/:date", func(c echo.Context) error {
		startDate, err := time.Parse("2006-01-02", c.Param("date"))
		if err != nil {
			jsonResp := ErrorResponse{"error", "Invalid date."}
			return c.JSON(http.StatusBadRequest, jsonResp)
		}
		endDate := startDate.Add(time.Hour * 24 * 7)
		rows, err := DB.Query("SELECT id, date, text FROM announcements WHERE date >= ? AND date <= ?", startDate.Format("2006-01-02"), endDate.Format("2006-01-02"))
		if err != nil {
			log.Println("Error while getting announcement information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		announcements := []PlannerAnnouncement{}
		for rows.Next() {
			resp := PlannerAnnouncement{-1, "", ""}
			rows.Scan(&resp.ID, &resp.Date, &resp.Text)
			announcements = append(announcements, resp)
		}
		jsonResp := MultiPlannerAnnouncementResponse{"ok", announcements}
		return c.JSON(http.StatusOK, jsonResp)
	})

	e.GET("/planner/fridays/get/:date", func(c echo.Context) error {
		rows, err := DB.Query("SELECT * FROM fridays WHERE date = ?", c.Param("date"))
		if err != nil {
			log.Println("Error while getting friday information: ")
			log.Println(err)
			jsonResp := StatusResponse{"error"}
			return c.JSON(http.StatusInternalServerError, jsonResp)
		}
		defer rows.Close()
		if rows.Next() {
			resp := PlannerFriday{-1, "", -1}
			rows.Scan(&resp.ID, &resp.Date, &resp.Index)
			jsonResp := PlannerFridayResponse{"ok", resp}
			return c.JSON(http.StatusOK, jsonResp)
		} else {
			jsonResp := StatusResponse{"ok"}
			return c.JSON(http.StatusOK, jsonResp)
		}
	})
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/go-martini/martini"
	conf "github.com/QubitProducts/bamboo/configuration"
	"github.com/QubitProducts/bamboo/services/service"
)

type ServiceAPI struct {
	Config  *conf.Configuration
	Storage service.Storage
}

func (d *ServiceAPI) All(w http.ResponseWriter, r *http.Request) {
	services, err := d.Storage.All()

	if err != nil {
		responseError(w, err.Error())
		return
	}

	byId := make(map[string]service.Service, len(services))
	for _, s := range services {
		byId[s.Id] = s
	}

	responseJSON(w, byId)
}

func (d *ServiceAPI) Create(w http.ResponseWriter, r *http.Request) {
	service, err := extractService(r)

	if err != nil {
		responseError(w, err.Error())
		return
	}

	err = d.Storage.Upsert(service)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, service)
}

func (d *ServiceAPI) Put(params martini.Params, w http.ResponseWriter, r *http.Request) {
	service, err := extractService(r)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	err = d.Storage.Upsert(service)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, service)
}

func (d *ServiceAPI) Delete(params martini.Params, w http.ResponseWriter, r *http.Request) {
	serviceId := params["_1"]
	err := d.Storage.Delete(serviceId)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, new(map[string]string))
}

func extractService(r *http.Request) (service.Service, error) {
	var serviceModel service.Service
	payload, _ := ioutil.ReadAll(r.Body)

	err := json.Unmarshal(payload, &serviceModel)
	if err != nil {
		return serviceModel, errors.New("Unable to decode JSON request")
	}
	if !strings.HasPrefix(serviceModel.Id, "/") {
		serviceModel.Id = "/" + serviceModel.Id
	}

	return serviceModel, nil
}

func responseError(w http.ResponseWriter, message string) {
	http.Error(w, message, http.StatusBadRequest)
}

func responseJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	bites, _ := json.Marshal(data)
	w.Write(bites)
}
<commit_msg>Prepend / to service ID in DELETE.<commit_after>package api

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/QubitProducts/bamboo/Godeps/_workspace/src/github.com/go-martini/martini"
	conf "github.com/QubitProducts/bamboo/configuration"
	"github.com/QubitProducts/bamboo/services/service"
)

type ServiceAPI struct {
	Config  *conf.Configuration
	Storage service.Storage
}

func (d *ServiceAPI) All(w http.ResponseWriter, r *http.Request) {
	services, err := d.Storage.All()

	if err != nil {
		responseError(w, err.Error())
		return
	}

	byId := make(map[string]service.Service, len(services))
	for _, s := range services {
		byId[s.Id] = s
	}

	responseJSON(w, byId)
}

func (d *ServiceAPI) Create(w http.ResponseWriter, r *http.Request) {
	service, err := extractService(r)

	if err != nil {
		responseError(w, err.Error())
		return
	}

	err = d.Storage.Upsert(service)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, service)
}

func (d *ServiceAPI) Put(params martini.Params, w http.ResponseWriter, r *http.Request) {
	service, err := extractService(r)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	err = d.Storage.Upsert(service)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, service)
}

func (d *ServiceAPI) Delete(params martini.Params, w http.ResponseWriter, r *http.Request) {
	serviceId := params["_1"]
	if !strings.HasPrefix(serviceId, "/") {
		serviceId = "/" + serviceId
	}

	err := d.Storage.Delete(serviceId)
	if err != nil {
		responseError(w, err.Error())
		return
	}

	responseJSON(w, new(map[string]string))
}

func extractService(r *http.Request) (service.Service, error) {
	var serviceModel service.Service
	payload, _ := ioutil.ReadAll(r.Body)

	err := json.Unmarshal(payload, &serviceModel)
	if err != nil {
		return serviceModel, errors.New("Unable to decode JSON request")
	}
	if !strings.HasPrefix(serviceModel.Id, "/") {
		serviceModel.Id = "/" + serviceModel.Id
	}

	return serviceModel, nil
}

func responseError(w http.ResponseWriter, message string) {
	http.Error(w, message, http.StatusBadRequest)
}

func responseJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	bites, _ := json.Marshal(data)
	w.Write(bites)
}
<|endoftext|>


<commit_before>package updater

import (
	"TOGY/config"
	"encoding/json"
	"io"
	"net/http"
)

type Info struct {
	Broadcast bool
	FileType  string
	Config    bool
}

func GetInfo(c *config.Config) (i Info, err error) {
	r, err := downloadInfo(c.UpdateURL + "/update?client=" + c.Name)
	if err != nil {
		return
	}
	defer r.Close()

	return parseInfo(r)
}

func downloadInfo(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

func parseInfo(r io.Reader) (i Info, err error) {
	d := json.NewDecoder(r)
	err = d.Decode(&i)
	return
}

func AnnounceBroadcast(c *config.Config) error {
	url := c.UpdateURL + "/presentation/active/downloadComplete?client=" + c.Name
	_, err := http.Get(url)
	return err
}

func AnnounceConfig(c *config.Config) error {
	url := c.UpdateURL + "/cofig/downloadComplete?client=" + c.Name
	_, err := http.Get(url)
	return err
}
<commit_msg>Removed an intentional typo from the final version<commit_after>package updater

import (
	"TOGY/config"
	"encoding/json"
	"io"
	"net/http"
)

type Info struct {
	Broadcast bool
	FileType  string
	Config    bool
}

func GetInfo(c *config.Config) (i Info, err error) {
	r, err := downloadInfo(c.UpdateURL + "/update?client=" + c.Name)
	if err != nil {
		return
	}
	defer r.Close()

	return parseInfo(r)
}

func downloadInfo(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	return resp.Body, nil
}

func parseInfo(r io.Reader) (i Info, err error) {
	d := json.NewDecoder(r)
	err = d.Decode(&i)
	return
}

func AnnounceBroadcast(c *config.Config) error {
	url := c.UpdateURL + "/presentation/active/downloadComplete?client=" + c.Name
	_, err := http.Get(url)
	return err
}

func AnnounceConfig(c *config.Config) error {
	url := c.UpdateURL + "/config/downloadComplete?client=" + c.Name
	_, err := http.Get(url)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"github.com/mitchellh/packer/packer"
	"testing"
)

// This is totally not isolated from the filesystem
func TestUpdateJsonFileWithString(t *testing.T) {
	file := "/tmp/testpathstring.json"
	paths := []string{"testpathstring"}
	ui := &packer.BasicUi{
		Reader:      new(bytes.Buffer),
		Writer:      new(bytes.Buffer),
		ErrorWriter: new(bytes.Buffer),
	}
	value := "simplevalue_string"
	_ = UpdateJsonFile(file, paths, value, ui, true)
}

func TestUpdateJsonFileWithMap(t *testing.T) {
	file := "/tmp/testpathmap.json"
	paths := []string{"testpathmap"}
	ui := &packer.BasicUi{
		Reader:      new(bytes.Buffer),
		Writer:      new(bytes.Buffer),
		ErrorWriter: new(bytes.Buffer),
	}
	value := map[string]string{"us-east-1": "ami-123456"}
	_ = UpdateJsonFile(file, paths, value, ui, true)
}
<commit_msg>Update test to reflect multi-region revert.<commit_after>package main

import (
	"bytes"
	"testing"

	"github.com/mitchellh/packer/packer"
	"github.com/stretchr/testify/assert"
)

func TestUpdateJSONFileShouldGiveErrorWhenFileDoesNotExist(t *testing.T) {
	file := "testdata/some_inexisting_file.json"
	paths := []string{"some.path.foo.bar"}
	ui := &packer.BasicUi{
		Reader:      new(bytes.Buffer),
		Writer:      new(bytes.Buffer),
		ErrorWriter: new(bytes.Buffer),
	}
	value := "ami-12345"
	err := UpdateJSONFile(file, paths, value, ui)
	assert.Equal(t, err.Error(), "open testdata/some_inexisting_file.json: no such file or directory")
}
<|endoftext|>


<commit_before>package url

type Type uint8

const (
	TypeSSH Type = iota
	TypePath
)

func Classify(raw string) Type {
	// On Windows, paths beginning with x:\ or x:/ (where x is a-z or A-Z) are
	// almost certainly referring to local paths, but will trigger the SCP URL
	// detection, so we have to check those first. This is, of course, something
	// of a heuristic, but we're unlikely to encounter 1-character hostnames and
	// very likely to encounter Windows paths (except on POSIX, where this check
	// always returns false). If Windows users do have a 1-character hostname,
	// they should just use some other addressing scheme for it (e.g. an IP
	// address or alternate hostname).
	if isWindowsPath(raw) {
		return TypePath
	}

	// Check if this is an SCP-style URL. A URL is classified as such if it
	// contains a colon with no forward slashes before it.
	for _, c := range raw {
		if c == ':' {
			return TypeSSH
		} else if c == '/' {
			break
		}
	}

	// Otherwise, assume this is a raw path.
	return TypePath
}
<commit_msg>Changed URL type enumeration order.<commit_after>package url

type Type uint8

const (
	TypePath Type = iota
	TypeSSH
)

func Classify(raw string) Type {
	// On Windows, paths beginning with x:\ or x:/ (where x is a-z or A-Z) are
	// almost certainly referring to local paths, but will trigger the SCP URL
	// detection, so we have to check those first. This is, of course, something
	// of a heuristic, but we're unlikely to encounter 1-character hostnames and
	// very likely to encounter Windows paths (except on POSIX, where this check
	// always returns false). If Windows users do have a 1-character hostname,
	// they should just use some other addressing scheme for it (e.g. an IP
	// address or alternate hostname).
	if isWindowsPath(raw) {
		return TypePath
	}

	// Check if this is an SCP-style URL. A URL is classified as such if it
	// contains a colon with no forward slashes before it.
	for _, c := range raw {
		if c == ':' {
			return TypeSSH
		} else if c == '/' {
			break
		}
	}

	// Otherwise, assume this is a raw path.
	return TypePath
}
<|endoftext|>


<commit_before>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/cred"
	"github.com/viant/toolbox/storage"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL           string   //URL of resource
	Credential    string   //name of file or alias to the file defined via credential service
	ParsedURL     *url.URL //parsed URL resource
	Cache         string   //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs int      //CacheExpiryMs expiry time in ms

	Name    string //name of a resource
	Version string //version of resource
	Type    string //resource type
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		Name:          r.Name,
		Version:       r.Version,
		URL:           r.URL,
		Type:          r.Type,
		Credential:    r.Credential,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//LoadCredentialload credential, returns username, password. It takes errorIfEmpty flag to return an error if there is issue with credential
func (r *Resource) LoadCredential(errorIfEmpty bool) (string, string, error) {
	if r.Credential == "" {
		if errorIfEmpty {
			return "", "", fmt.Errorf("Credential was empty: %v", r.Credential)
		}
		return "", "", nil
	}
	credential, err := cred.NewConfig(r.Credential)
	if err != nil {
		return "", "", fmt.Errorf("Failed to load credential: %v %v", r.Credential, err)
	}
	return credential.Username, credential.Password, nil
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}

	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}

	service, err := storage.NewServiceForURL(r.URL, r.Credential)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) Decode(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("Fail to %T decode on empty resource", decoderFactory)
	}
	if r == nil {
		return fmt.Errorf("Fail to decode %v,  decoderFactory was empty", r.URL, decoderFactory)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}
	return decoderFactory.Create(bytes.NewReader(content)).Decode(target)
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.Decode(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	return r.Decode(target, toolbox.NewYamlDecoderFactory())
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			if toolbox.FileExists(candidate) {
				URL = candidate
			}
		}
	}
	return toolbox.FileSchema + URL
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		ParsedURL:     parsedURL,
		URL:           URL,
		Credential:    credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<commit_msg>added rename<commit_after>package url

import (
	"bytes"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/cred"
	"github.com/viant/toolbox/storage"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

//Resource represents a URL based resource, with enriched meta info
type Resource struct {
	URL           string   //URL of resource
	Credential    string   //name of file or alias to the file defined via credential service
	ParsedURL     *url.URL //parsed URL resource
	Cache         string   //Cache path for the resource, if specified resource will be cached in the specified path
	CacheExpiryMs int      //CacheExpiryMs expiry time in ms

	Name    string //name of a resource
	Version string //version of resource
	Type    string //resource type
}

//Clone creates a clone of the resource
func (r *Resource) Clone() *Resource {
	return &Resource{
		Name:          r.Name,
		Version:       r.Version,
		URL:           r.URL,
		Type:          r.Type,
		Credential:    r.Credential,
		ParsedURL:     r.ParsedURL,
		Cache:         r.Cache,
		CacheExpiryMs: r.CacheExpiryMs,
	}
}

var defaultSchemePorts = map[string]int{
	"ssh":   22,
	"http":  80,
	"https": 443,
}

//Host returns url's host name with user name if user name is part of url
func (r *Resource) Host() string {
	result := r.ParsedURL.Hostname() + ":" + r.Port()
	if r.ParsedURL.User != nil {
		result = r.ParsedURL.User.Username() + "@" + result
	}
	return result
}

//Port returns url's port
func (r *Resource) Port() string {
	port := r.ParsedURL.Port()
	if port == "" && r.ParsedURL != nil {
		if value, ok := defaultSchemePorts[r.ParsedURL.Scheme]; ok {
			port = toolbox.AsString(value)
		}
	}
	return port
}

//LoadCredentialload credential, returns username, password. It takes errorIfEmpty flag to return an error if there is issue with credential
func (r *Resource) LoadCredential(errorIfEmpty bool) (string, string, error) {
	if r.Credential == "" {
		if errorIfEmpty {
			return "", "", fmt.Errorf("Credential was empty: %v", r.Credential)
		}
		return "", "", nil
	}
	credential, err := cred.NewConfig(r.Credential)
	if err != nil {
		return "", "", fmt.Errorf("Failed to load credential: %v %v", r.Credential, err)
	}
	return credential.Username, credential.Password, nil
}

//Download downloads data from URL, it returns data as []byte, or error, if resource is cacheable it first look into cache
func (r *Resource) Download() ([]byte, error) {
	if r == nil {
		return nil, fmt.Errorf("Fail to download content on empty resource")
	}

	if r.Cachable() {
		content := r.readFromCache()
		if content != nil {
			return content, nil
		}
	}

	service, err := storage.NewServiceForURL(r.URL, r.Credential)
	if err != nil {
		return nil, err
	}
	object, err := service.StorageObject(r.URL)
	if err != nil {
		return nil, err
	}
	reader, err := service.Download(object)
	if err != nil {
		return nil, err
	}
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return nil, err
	}
	if r.Cachable() {
		_ = ioutil.WriteFile(r.Cache, content, 0666)
	}
	return content, err
}

//DownloadText returns a text downloaded from url
func (r *Resource) DownloadText() (string, error) {
	var result, err = r.Download()
	if err != nil {
		return "", err
	}
	return string(result), err
}

//Decode decodes url's data into target, it takes decoderFactory which decodes data into target
func (r *Resource) Decode(target interface{}, decoderFactory toolbox.DecoderFactory) error {
	if r == nil {
		return fmt.Errorf("Fail to %T decode on empty resource", decoderFactory)
	}
	if r == nil {
		return fmt.Errorf("Fail to decode %v,  decoderFactory was empty", r.URL, decoderFactory)
	}
	var content, err = r.Download()
	if err != nil {
		return err
	}
	return decoderFactory.Create(bytes.NewReader(content)).Decode(target)
}

//Rename renames URI name of this resource
func (r *Resource) Rename(name string) (err error) {
	var _, currentName = toolbox.URLSplit(r.URL)
	r.URL = strings.Replace(r.URL, currentName, name, 1)
	r.ParsedURL, err = url.Parse(r.URL)
	return err
}

//JSONDecode decodes json resource into target
func (r *Resource) JSONDecode(target interface{}) error {
	return r.Decode(target, toolbox.NewJSONDecoderFactory())
}

//JSONDecode decodes yaml resource into target
func (r *Resource) YAMLDecode(target interface{}) error {
	return r.Decode(target, toolbox.NewYamlDecoderFactory())
}

func (r *Resource) readFromCache() []byte {
	if toolbox.FileExists(r.Cache) {
		info, err := os.Stat(r.Cache)
		var isExpired = false
		if err == nil && r.CacheExpiryMs > 0 {
			elapsed := time.Now().Sub(info.ModTime())
			isExpired = elapsed > time.Second*time.Duration(r.CacheExpiryMs)
		}
		content, err := ioutil.ReadFile(r.Cache)
		if err == nil && !isExpired {
			return content
		}
	}
	return nil
}

//Cachable returns true if resource is cachable
func (r *Resource) Cachable() bool {
	return r.Cache != ""
}

func normalizeURL(URL string) string {
	if strings.Contains(URL, "://") {
		return URL
	}
	if !strings.HasPrefix(URL, "/") {
		currentDirectory, err := os.Getwd()
		if err == nil {
			candidate := path.Join(currentDirectory, URL)
			if toolbox.FileExists(candidate) {
				URL = candidate
			}
		}
	}
	return toolbox.FileSchema + URL
}

//NewResource returns a new resource for provided URL, followed by optional credential, cache and cache expiryMs.
func NewResource(Params ...interface{}) *Resource {
	if len(Params) == 0 {
		return nil
	}
	var URL = toolbox.AsString(Params[0])
	URL = normalizeURL(URL)

	var credential string
	if len(Params) > 1 {
		credential = toolbox.AsString(Params[1])
	}
	var cache string
	if len(Params) > 2 {
		cache = toolbox.AsString(Params[2])
	}
	var cacheExpiryMs int
	if len(Params) > 3 {
		cacheExpiryMs = toolbox.AsInt(Params[3])
	}
	parsedURL, _ := url.Parse(URL)
	return &Resource{
		ParsedURL:     parsedURL,
		URL:           URL,
		Credential:    credential,
		Cache:         cache,
		CacheExpiryMs: cacheExpiryMs,
	}
}
<|endoftext|>


<commit_before>package version

var (
	// The git commit that was compiled. This will be filled in by the compiler.
	GitCommit   string
	GitDescribe string

	// Whether cgo is enabled or not; set at build time
	CgoEnabled bool

	Version           = "1.2.0"
	VersionPrerelease = "beta1"
	VersionMetadata   = ""
)
<commit_msg>Bump version in sdk<commit_after>package version

var (
	// The git commit that was compiled. This will be filled in by the compiler.
	GitCommit   string
	GitDescribe string

	// Whether cgo is enabled or not; set at build time
	CgoEnabled bool

	Version           = "1.2.0"
	VersionPrerelease = "beta2"
	VersionMetadata   = ""
)
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"os"
)

const (
	// LvlOutline shows client version & arguments passed in, HTTP URLs & status codes, subprocess calls.
	LvlOutline = 1 + iota
	// LvlArgs shows args going in to library functions
	LvlArgs
	// LvlFlags is used to show/hide the way flags and command line arguments are being messed with in main
	LvlFlags
	// LvlHTTPData is used to show/hide raw HTTP request and response bodies.
	LvlHTTPData
	// LvlMisc is used for any other minutiae
	LvlMisc
)

// DebugLevel determines whether or not debugging output should be output to stderr.
var DebugLevel int

// LogFile is the file which bytemark-client is to log to. This can be nil, in which case it won't. Usually ~/.bytemark/debug.log
var LogFile *os.File

// Error outputs stuff to os.Stderr and LogFile, one thing per line.
func Error(stuff ...interface{}) {
	if len(stuff) == 0 {
		Error("")
	}
	for _, v := range stuff {
		_, _ = fmt.Fprintln(os.Stderr, v)
		if LogFile != nil {
			_, _ = fmt.Fprintln(LogFile, v)
		}
	}

}

// Errorf formats the string and outputs it to Stderr and Logfile.
func Errorf(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format, args...)
	if LogFile != nil {
		_, _ = fmt.Fprintf(LogFile, format, args...)
	}
}

// Log outputs stuff to os.Stderr and LogFile, one thing per line.
func Log(stuff ...interface{}) {
	if len(stuff) == 0 {
		Log("")
	}
	for _, v := range stuff {
		_, _ = fmt.Fprintln(os.Stderr, v)
		if LogFile != nil {
			_, _ = fmt.Fprintln(LogFile, v)
		}
	}
}

// Logf formats the string and outputs it to Stderr and Logfile.
func Logf(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format, args...)
	if LogFile != nil {
		_, _ = fmt.Fprintf(LogFile, format, args...)
	}
}

// Output outputs stuff to os.Stdout and LogFile, one thing per line.
func Output(stuff ...interface{}) {

	if len(stuff) == 0 {
		Output("")
	}
	for _, v := range stuff {
		fmt.Println(v)
		if LogFile != nil {
			_, _ = fmt.Fprintln(LogFile, v)
		}
	}
}

// Outputf formats the string and outputs it to Stdout and Logfile.
func Outputf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	if LogFile != nil {
		_, _ = fmt.Fprintf(LogFile, format, args...)
	}
}

// Debug outputs stuff to LogFile, and to Stderr if DebugLevel >= level. One thing per line.
func Debug(level int, stuff ...interface{}) {
	for _, v := range stuff {
		if level <= DebugLevel {
			_, _ = fmt.Fprintln(os.Stderr, v)
		}
		if LogFile != nil {
			_, _ = fmt.Fprintln(LogFile, v)
		}
	}
}

// Debugf formats the string and outputs it to LogFile, and to Stderr if DebugLevel >= level.
func Debugf(level int, format string, args ...interface{}) {
	if level <= DebugLevel {
		_, _ = fmt.Fprintf(os.Stderr, format, args...)
	}
	if LogFile != nil {
		_, _ = fmt.Fprintf(LogFile, format, args...)
	}
}
<commit_msg>Revert "Ignore errors writing to stdout/stderr/log file"<commit_after>package log

import (
	"fmt"
	"os"
)

const (
	// LvlOutline shows client version & arguments passed in, HTTP URLs & status codes, subprocess calls.
	LvlOutline = 1 + iota
	// LvlArgs shows args going in to library functions
	LvlArgs
	// LvlFlags is used to show/hide the way flags and command line arguments are being messed with in main
	LvlFlags
	// LvlHTTPData is used to show/hide raw HTTP request and response bodies.
	LvlHTTPData
	// LvlMisc is used for any other minutiae
	LvlMisc
)

// DebugLevel determines whether or not debugging output should be output to stderr.
var DebugLevel int

// LogFile is the file which bytemark-client is to log to. This can be nil, in which case it won't. Usually ~/.bytemark/debug.log
var LogFile *os.File

// Error outputs stuff to os.Stderr and LogFile, one thing per line.
func Error(stuff ...interface{}) {
	if len(stuff) == 0 {
		Error("")
	}
	for _, v := range stuff {
		fmt.Fprintln(os.Stderr, v)
		if LogFile != nil {
			fmt.Fprintln(LogFile, v)
		}
	}

}

// Errorf formats the string and outputs it to Stderr and Logfile.
func Errorf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	if LogFile != nil {
		fmt.Fprintf(LogFile, format, args...)
	}
}

// Log outputs stuff to os.Stderr and LogFile, one thing per line.
func Log(stuff ...interface{}) {
	if len(stuff) == 0 {
		Log("")
	}
	for _, v := range stuff {
		fmt.Fprintln(os.Stderr, v)
		if LogFile != nil {
			fmt.Fprintln(LogFile, v)
		}
	}
}

// Logf formats the string and outputs it to Stderr and Logfile.
func Logf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	if LogFile != nil {
		fmt.Fprintf(LogFile, format, args...)
	}
}

// Output outputs stuff to os.Stdout and LogFile, one thing per line.
func Output(stuff ...interface{}) {

	if len(stuff) == 0 {
		Output("")
	}
	for _, v := range stuff {
		fmt.Println(v)
		if LogFile != nil {
			fmt.Fprintln(LogFile, v)
		}
	}
}

// Outputf formats the string and outputs it to Stdout and Logfile.
func Outputf(format string, args ...interface{}) {
	fmt.Printf(format, args...)
	if LogFile != nil {
		fmt.Fprintf(LogFile, format, args...)
	}
}

// Debug outputs stuff to LogFile, and to Stderr if DebugLevel >= level. One thing per line.
func Debug(level int, stuff ...interface{}) {
	for _, v := range stuff {
		if level <= DebugLevel {
			fmt.Fprintln(os.Stderr, v)
		}
		if LogFile != nil {
			fmt.Fprintln(LogFile, v)
		}
	}
}

// Debugf formats the string and outputs it to LogFile, and to Stderr if DebugLevel >= level.
func Debugf(level int, format string, args ...interface{}) {
	if level <= DebugLevel {
		fmt.Fprintf(os.Stderr, format, args...)
	}
	if LogFile != nil {
		fmt.Fprintf(LogFile, format, args...)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package utils

import (
	l4g "code.google.com/p/log4go"
	"encoding/json"
	"net/mail"
	"os"
	"path/filepath"
)

const (
	MODE_DEV  = "dev"
	MODE_BETA = "beta"
	MODE_PROD = "prod"
)

type ServiceSettings struct {
	SiteName       string
	Domain         string
	Mode           string
	AllowTesting   bool
	UseSSL         bool
	Port           string
	Version        string
	Shards         map[string]string
	InviteSalt     string
	PublicLinkSalt string
	ResetSalt      string
	AnalyticsUrl   string
}

type SqlSettings struct {
	DriverName         string
	DataSource         string
	DataSourceReplicas []string
	MaxIdleConns       int
	MaxOpenConns       int
	Trace              bool
	AtRestEncryptKey   string
}

type RedisSettings struct {
	DataSource   string
	MaxOpenConns int
}

type LogSettings struct {
	ConsoleEnable bool
	ConsoleLevel  string
	FileEnable    bool
	FileLevel     string
	FileFormat    string
	FileLocation  string
}

type AWSSettings struct {
	S3AccessKeyId          string
	S3SecretAccessKey      string
	S3Bucket               string
	S3Region               string
	Route53AccessKeyId     string
	Route53SecretAccessKey string
	Route53ZoneId          string
	Route53Region          string
}

type ImageSettings struct {
	ThumbnailWidth  uint
	ThumbnailHeight uint
	PreviewWidth    uint
	PreviewHeight   uint
	ProfileWidth    uint
	ProfileHeight   uint
}

type EmailSettings struct {
	SMTPUsername         string
	SMTPPassword         string
	SMTPServer           string
	UseTLS               bool
	FeedbackEmail        string
	FeedbackName         string
	ApplePushServer      string
	ApplePushCertPublic  string
	ApplePushCertPrivate string
}

type PrivacySettings struct {
	ShowEmailAddress bool
	ShowPhoneNumber  bool
	ShowSkypeId      bool
	ShowFullName     bool
}

type TeamSettings struct {
	MaxUsersPerTeam   int
	AllowPublicLink   bool
	AllowValet        bool
	TermsLink         string
	PrivacyLink       string
	AboutLink         string
	HelpLink          string
	ReportProblemLink string
	TourLink          string
	DefaultThemeColor string
}

type Config struct {
	LogSettings     LogSettings
	ServiceSettings ServiceSettings
	SqlSettings     SqlSettings
	RedisSettings   RedisSettings
	AWSSettings     AWSSettings
	ImageSettings   ImageSettings
	EmailSettings   EmailSettings
	PrivacySettings PrivacySettings
	TeamSettings    TeamSettings
}

func (o *Config) ToJson() string {
	b, err := json.Marshal(o)
	if err != nil {
		return ""
	} else {
		return string(b)
	}
}

var Cfg *Config = &Config{}
var SanitizeOptions map[string]bool = map[string]bool{}

func findConfigFile(fileName string) string {
	if _, err := os.Stat("/tmp/" + fileName); err == nil {
		fileName, _ = filepath.Abs("/tmp/" + fileName)
	} else if _, err := os.Stat("./config/" + fileName); err == nil {
		fileName, _ = filepath.Abs("./config/" + fileName)
	} else if _, err := os.Stat("../config/" + fileName); err == nil {
		fileName, _ = filepath.Abs("../config/" + fileName)
	} else if _, err := os.Stat(fileName); err == nil {
		fileName, _ = filepath.Abs(fileName)
	}

	return fileName
}

func FindDir(dir string) string {
	fileName := "."
	if _, err := os.Stat("./" + dir + "/"); err == nil {
		fileName, _ = filepath.Abs("./" + dir + "/")
	} else if _, err := os.Stat("../" + dir + "/"); err == nil {
		fileName, _ = filepath.Abs("../" + dir + "/")
	} else if _, err := os.Stat("/tmp/" + dir); err == nil {
		fileName, _ = filepath.Abs("/tmp/" + dir)
	}

	return fileName + "/"
}

func configureLog(s LogSettings) {

	l4g.Close()

	if s.ConsoleEnable {
		level := l4g.DEBUG
		if s.ConsoleLevel == "INFO" {
			level = l4g.INFO
		} else if s.ConsoleLevel == "ERROR" {
			level = l4g.ERROR
		}

		l4g.AddFilter("stdout", level, l4g.NewConsoleLogWriter())
	}

	if s.FileEnable {
		if s.FileFormat == "" {
			s.FileFormat = "[%D %T] [%L] %M"
		}

		if s.FileLocation == "" {
			s.FileLocation = FindDir("logs") + "mattermost.log"
		}

		level := l4g.DEBUG
		if s.FileLevel == "INFO" {
			level = l4g.INFO
		} else if s.FileLevel == "ERROR" {
			level = l4g.ERROR
		}

		flw := l4g.NewFileLogWriter(s.FileLocation, false)
		flw.SetFormat(s.FileFormat)
		flw.SetRotate(true)
		flw.SetRotateLines(100000)
		l4g.AddFilter("file", level, flw)
	}
}

// LoadConfig will try to search around for the corresponding config file.
// It will search /tmp/fileName then attempt ./config/fileName,
// then ../config/fileName and last it will look at fileName
func LoadConfig(fileName string) {

	fileName = findConfigFile(fileName)
	l4g.Info("Loading config file at " + fileName)

	file, err := os.Open(fileName)
	if err != nil {
		panic("Error opening config file=" + fileName + ", err=" + err.Error())
	}

	decoder := json.NewDecoder(file)
	config := Config{}
	err = decoder.Decode(&config)
	if err != nil {
		panic("Error decoding configuration " + err.Error())
	}

	// Grabs the domain from enviroment variable if not in configuration
	if config.ServiceSettings.Domain == "" {
		config.ServiceSettings.Domain = os.Getenv("MATTERMOST_DOMAIN")
	}

	// Check for a valid email for feedback, if not then do feedback@domain
	if _, err := mail.ParseAddress(config.EmailSettings.FeedbackEmail); err != nil {
		config.EmailSettings.FeedbackEmail = "feedback@" + config.ServiceSettings.Domain
	}

	configureLog(config.LogSettings)

	Cfg = &config
	SanitizeOptions = getSanitizeOptions()

	// Validates our mail settings
	if err := CheckMailSettings(); err != nil {
		l4g.Error("Email settings are not valid err=%v", err)
	}
}

func getSanitizeOptions() map[string]bool {
	options := map[string]bool{}
	options["fullname"] = Cfg.PrivacySettings.ShowFullName
	options["email"] = Cfg.PrivacySettings.ShowEmailAddress
	options["skypeid"] = Cfg.PrivacySettings.ShowSkypeId
	options["phonenumber"] = Cfg.PrivacySettings.ShowPhoneNumber

	return options
}

func IsS3Configured() bool {
	if Cfg.AWSSettings.S3AccessKeyId == "" || Cfg.AWSSettings.S3SecretAccessKey == "" || Cfg.AWSSettings.S3Region == "" || Cfg.AWSSettings.S3Bucket == "" {
		return false
	}

	return true
}
<commit_msg>Defaulting to domain of localhost when no domain is given<commit_after>// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
// See License.txt for license information.

package utils

import (
	l4g "code.google.com/p/log4go"
	"encoding/json"
	"net/mail"
	"os"
	"path/filepath"
)

const (
	MODE_DEV  = "dev"
	MODE_BETA = "beta"
	MODE_PROD = "prod"
)

type ServiceSettings struct {
	SiteName       string
	Domain         string
	Mode           string
	AllowTesting   bool
	UseSSL         bool
	Port           string
	Version        string
	Shards         map[string]string
	InviteSalt     string
	PublicLinkSalt string
	ResetSalt      string
	AnalyticsUrl   string
}

type SqlSettings struct {
	DriverName         string
	DataSource         string
	DataSourceReplicas []string
	MaxIdleConns       int
	MaxOpenConns       int
	Trace              bool
	AtRestEncryptKey   string
}

type RedisSettings struct {
	DataSource   string
	MaxOpenConns int
}

type LogSettings struct {
	ConsoleEnable bool
	ConsoleLevel  string
	FileEnable    bool
	FileLevel     string
	FileFormat    string
	FileLocation  string
}

type AWSSettings struct {
	S3AccessKeyId          string
	S3SecretAccessKey      string
	S3Bucket               string
	S3Region               string
	Route53AccessKeyId     string
	Route53SecretAccessKey string
	Route53ZoneId          string
	Route53Region          string
}

type ImageSettings struct {
	ThumbnailWidth  uint
	ThumbnailHeight uint
	PreviewWidth    uint
	PreviewHeight   uint
	ProfileWidth    uint
	ProfileHeight   uint
}

type EmailSettings struct {
	SMTPUsername         string
	SMTPPassword         string
	SMTPServer           string
	UseTLS               bool
	FeedbackEmail        string
	FeedbackName         string
	ApplePushServer      string
	ApplePushCertPublic  string
	ApplePushCertPrivate string
}

type PrivacySettings struct {
	ShowEmailAddress bool
	ShowPhoneNumber  bool
	ShowSkypeId      bool
	ShowFullName     bool
}

type TeamSettings struct {
	MaxUsersPerTeam   int
	AllowPublicLink   bool
	AllowValet        bool
	TermsLink         string
	PrivacyLink       string
	AboutLink         string
	HelpLink          string
	ReportProblemLink string
	TourLink          string
	DefaultThemeColor string
}

type Config struct {
	LogSettings     LogSettings
	ServiceSettings ServiceSettings
	SqlSettings     SqlSettings
	RedisSettings   RedisSettings
	AWSSettings     AWSSettings
	ImageSettings   ImageSettings
	EmailSettings   EmailSettings
	PrivacySettings PrivacySettings
	TeamSettings    TeamSettings
}

func (o *Config) ToJson() string {
	b, err := json.Marshal(o)
	if err != nil {
		return ""
	} else {
		return string(b)
	}
}

var Cfg *Config = &Config{}
var SanitizeOptions map[string]bool = map[string]bool{}

func findConfigFile(fileName string) string {
	if _, err := os.Stat("/tmp/" + fileName); err == nil {
		fileName, _ = filepath.Abs("/tmp/" + fileName)
	} else if _, err := os.Stat("./config/" + fileName); err == nil {
		fileName, _ = filepath.Abs("./config/" + fileName)
	} else if _, err := os.Stat("../config/" + fileName); err == nil {
		fileName, _ = filepath.Abs("../config/" + fileName)
	} else if _, err := os.Stat(fileName); err == nil {
		fileName, _ = filepath.Abs(fileName)
	}

	return fileName
}

func FindDir(dir string) string {
	fileName := "."
	if _, err := os.Stat("./" + dir + "/"); err == nil {
		fileName, _ = filepath.Abs("./" + dir + "/")
	} else if _, err := os.Stat("../" + dir + "/"); err == nil {
		fileName, _ = filepath.Abs("../" + dir + "/")
	} else if _, err := os.Stat("/tmp/" + dir); err == nil {
		fileName, _ = filepath.Abs("/tmp/" + dir)
	}

	return fileName + "/"
}

func configureLog(s LogSettings) {

	l4g.Close()

	if s.ConsoleEnable {
		level := l4g.DEBUG
		if s.ConsoleLevel == "INFO" {
			level = l4g.INFO
		} else if s.ConsoleLevel == "ERROR" {
			level = l4g.ERROR
		}

		l4g.AddFilter("stdout", level, l4g.NewConsoleLogWriter())
	}

	if s.FileEnable {
		if s.FileFormat == "" {
			s.FileFormat = "[%D %T] [%L] %M"
		}

		if s.FileLocation == "" {
			s.FileLocation = FindDir("logs") + "mattermost.log"
		}

		level := l4g.DEBUG
		if s.FileLevel == "INFO" {
			level = l4g.INFO
		} else if s.FileLevel == "ERROR" {
			level = l4g.ERROR
		}

		flw := l4g.NewFileLogWriter(s.FileLocation, false)
		flw.SetFormat(s.FileFormat)
		flw.SetRotate(true)
		flw.SetRotateLines(100000)
		l4g.AddFilter("file", level, flw)
	}
}

// LoadConfig will try to search around for the corresponding config file.
// It will search /tmp/fileName then attempt ./config/fileName,
// then ../config/fileName and last it will look at fileName
func LoadConfig(fileName string) {

	fileName = findConfigFile(fileName)
	l4g.Info("Loading config file at " + fileName)

	file, err := os.Open(fileName)
	if err != nil {
		panic("Error opening config file=" + fileName + ", err=" + err.Error())
	}

	decoder := json.NewDecoder(file)
	config := Config{}
	err = decoder.Decode(&config)
	if err != nil {
		panic("Error decoding configuration " + err.Error())
	}

	// Grabs the domain from enviroment variable if not in configuration
	if config.ServiceSettings.Domain == "" {
		config.ServiceSettings.Domain = os.Getenv("MATTERMOST_DOMAIN")
		// If the enviroment variable is not set, use a default
		if config.ServiceSettings.Domain == "" {
			config.ServiceSettings.Domain = "localhost"
		}
	}

	// Check for a valid email for feedback, if not then do feedback@domain
	if _, err := mail.ParseAddress(config.EmailSettings.FeedbackEmail); err != nil {
		config.EmailSettings.FeedbackEmail = "feedback@" + config.ServiceSettings.Domain
	}

	configureLog(config.LogSettings)

	Cfg = &config
	SanitizeOptions = getSanitizeOptions()

	// Validates our mail settings
	if err := CheckMailSettings(); err != nil {
		l4g.Error("Email settings are not valid err=%v", err)
	}
}

func getSanitizeOptions() map[string]bool {
	options := map[string]bool{}
	options["fullname"] = Cfg.PrivacySettings.ShowFullName
	options["email"] = Cfg.PrivacySettings.ShowEmailAddress
	options["skypeid"] = Cfg.PrivacySettings.ShowSkypeId
	options["phonenumber"] = Cfg.PrivacySettings.ShowPhoneNumber

	return options
}

func IsS3Configured() bool {
	if Cfg.AWSSettings.S3AccessKeyId == "" || Cfg.AWSSettings.S3SecretAccessKey == "" || Cfg.AWSSettings.S3Region == "" || Cfg.AWSSettings.S3Bucket == "" {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package v3

import (
	"fmt"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	. "github.com/cloudfoundry/cf-acceptance-tests/helpers/v3_helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("v3 tasks", func() {
	var (
		appName                         string
		appGuid                         string
		packageGuid                     string
		spaceGuid                       string
		appCreationEnvironmentVariables string
	)

	BeforeEach(func() {
		appName = generator.PrefixedRandomName("CATS-APP-")
		spaceGuid = GetSpaceGuidFromName(context.RegularUserContext().Space)
		appCreationEnvironmentVariables = `"foo"=>"bar"`
		appGuid = CreateApp(appName, spaceGuid, `{"foo":"bar"}`)
		packageGuid = CreatePackage(appGuid)
		token := GetAuthToken()
		uploadUrl := fmt.Sprintf("%s/v3/packages/%s/upload", config.ApiEndpoint, packageGuid)
		UploadPackage(uploadUrl, assets.NewAssets().DoraZip, token)
		WaitForPackageToBeReady(packageGuid)
		dropletGuid := StagePackage(packageGuid, "{}")
		WaitForDropletToStage(dropletGuid)
		AssignDropletToApp(appGuid, dropletGuid)
	})

	AfterEach(func() {
		DeleteApp(appGuid)
	})

	XIt("can create a task", func() {
		config := helpers.LoadConfig()
		if config.IncludeTasks {
			postBody := `{"command": "echo 0", "name": "mreow"}`
			createCommand := cf.Cf("curl", fmt.Sprintf("/v3/apps/%s/tasks", appGuid), "-X", "POST", "-d", postBody).Wait(DEFAULT_TIMEOUT)
			Expect(createCommand).To(Exit(0))
			Expect(createCommand.Out.Contents()).To(ContainSubstring("echo 0"))
			Expect(createCommand.Out.Contents()).To(ContainSubstring("mreow"))
			Expect(createCommand.Out.Contents()).To(ContainSubstring("RUNNING"))
		}
	})
})
<commit_msg>Un-pend v3 tasks CAT<commit_after>package v3

import (
	"fmt"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
	. "github.com/cloudfoundry/cf-acceptance-tests/helpers/v3_helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("v3 tasks", func() {
	config := helpers.LoadConfig()
	if config.IncludeTasks {
		Context("tasks lifecycle", func() {
			var (
				appName                         string
				appGuid                         string
				packageGuid                     string
				spaceGuid                       string
				appCreationEnvironmentVariables string
			)

			BeforeEach(func() {
				appName = generator.PrefixedRandomName("CATS-APP-")
				spaceGuid = GetSpaceGuidFromName(context.RegularUserContext().Space)
				appCreationEnvironmentVariables = `"foo"=>"bar"`
				appGuid = CreateApp(appName, spaceGuid, `{"foo":"bar"}`)
				packageGuid = CreatePackage(appGuid)
				token := GetAuthToken()
				uploadUrl := fmt.Sprintf("%s/v3/packages/%s/upload", config.ApiEndpoint, packageGuid)
				UploadPackage(uploadUrl, assets.NewAssets().DoraZip, token)
				WaitForPackageToBeReady(packageGuid)
				dropletGuid := StagePackage(packageGuid, "{}")
				WaitForDropletToStage(dropletGuid)
				AssignDropletToApp(appGuid, dropletGuid)
			})

			AfterEach(func() {
				DeleteApp(appGuid)
			})

			It("can create a task", func() {
				postBody := `{"command": "echo 0", "name": "mreow"}`
				createCommand := cf.Cf("curl", fmt.Sprintf("/v3/apps/%s/tasks", appGuid), "-X", "POST", "-d", postBody).Wait(DEFAULT_TIMEOUT)
				Expect(createCommand).To(Exit(0))
				Expect(createCommand.Out.Contents()).To(ContainSubstring("echo 0"))
				Expect(createCommand.Out.Contents()).To(ContainSubstring("mreow"))
				Expect(createCommand.Out.Contents()).To(ContainSubstring("RUNNING"))
			})
		})
		}
	})
<|endoftext|>


<commit_before>// Copyright 2017-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package launch

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/datapath/route"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/endpoint/connector"
	"github.com/cilium/cilium/pkg/endpointmanager"
	healthPkg "github.com/cilium/cilium/pkg/health/client"
	healthDefaults "github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/launcher"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/pidfile"

	"github.com/vishvananda/netlink"
)

const (
	ciliumHealth = "cilium-health"
)

var (
	// vethName is the host-side link device name for cilium-health EP.
	vethName = "cilium_health"

	// vethPeerName is the endpoint-side link device name for cilium-health.
	vethPeerName = "cilium"

	// PidfilePath
	PidfilePath = "health-endpoint.pid"

	// client is used to ping the cilium-health endpoint as a health check.
	client *healthPkg.Client

	// NodeEpAnnotator is used to annotate nodes and pods in a cluster with
	// information about this cilium-health instance.
	NodeEpAnnotator Annotator
)

func logFromCommand(cmd *exec.Cmd, netns string) error {
	scopedLog := log.WithField("netns", netns)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	go func() {
		in := bufio.NewScanner(stdout)
		for in.Scan() {
			scopedLog.Debugf("%s", in.Text())
		}
	}()
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	go func() {
		in := bufio.NewScanner(stderr)
		for in.Scan() {
			scopedLog.Infof("%s", in.Text())
		}
	}()

	return nil
}

func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	routes := []route.Route{}

	if option.Config.EnableIPv4 {
		v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU())
		if err == nil {
			routes = append(routes, v4Routes...)
		} else {
			log.Debugf("Couldn't get IPv4 routes for health routing")
		}
	}

	if option.Config.EnableIPv6 {
		v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU())
		if err != nil {
			return fmt.Errorf("Failed to get IPv6 routes")
		}
		routes = append(routes, v6Routes...)
	}

	prog := "ip"
	args := []string{"netns", "exec", netns, "bash", "-c"}
	routeCmds := []string{}
	for _, rt := range routes {
		cmd := strings.Join(rt.ToIPCommand(dev), " ")
		log.WithField("netns", netns).WithField("command", cmd).Info("Adding route")
		routeCmds = append(routeCmds, cmd)
	}
	cmd := strings.Join(routeCmds, " && ")
	args = append(args, cmd)

	log.Debugf("Running \"%s %+v\"", prog, args)
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err == nil && len(out) > 0 {
		log.Warn(out)
	}

	return err
}

// PingEndpoint attempts to make an API ping request to the local cilium-health
// endpoint, and returns whether this was successful.
//
// This function must only be used from the same goroutine as LaunchAsEndpoint().
// It is safe to call PingEndpoint() before LaunchAsEndpoint() so long as the
// goroutine is the same for both calls.
func PingEndpoint() error {
	// client is shared with LaunchAsEndpoint().
	if client == nil {
		return fmt.Errorf("cilium-health endpoint hasn't yet been initialized")
	}
	_, err := client.Restapi.GetHello(nil)
	return err
}

// KillEndpoint attempts to kill any existing cilium-health endpoint if it
// exists.
//
// This is intended to be invoked in multiple situations:
// * The health endpoint has never been run before
// * The health endpoint was run during a previous run of the Cilium agent
// * The health endpoint crashed during the current run of the Cilium agent
//   and needs to be cleaned up before it is restarted.
func KillEndpoint() {
	path := filepath.Join(option.Config.StateDir, PidfilePath)
	if err := pidfile.Kill(path); err != nil {
		scopedLog := log.WithField(logfields.Path, path).WithError(err)
		scopedLog.Info("Failed to kill previous cilium-health instance")
	}
}

// CleanupEndpoint cleans up remaining resources associated with the health
// endpoint.
//
// This is expected to be called after the process is killed and the endpoint
// is removed from the endpointmanager.
func CleanupEndpoint() {
	scopedLog := log.WithField(logfields.Veth, vethName)
	if link, err := netlink.LinkByName(vethName); err == nil {
		err = netlink.LinkDel(link)
		if err != nil {
			scopedLog.WithError(err).Info("Couldn't delete cilium-health device")
		}
	} else {
		scopedLog.WithError(err).Debug("Didn't find existing device")
	}
}

// Annotator is an interface which describes anything which annotates a node
// with cilium-health metadata.
type Annotator interface {
	AnnotateNode(nodeName string, v4CIDR, v6CIDR *net.IPNet, v4HealthIP, v6HealthIP, v4CiliumHostIP net.IP) error
	AnnotatePod(k8sNamespace, k8sPodName, annotationKey, annotationValue string) error
}

// LaunchAsEndpoint launches the cilium-health agent in a nested network
// namespace and attaches it to Cilium the same way as any other endpoint,
// but with special reserved labels.
//
// CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure
// cleanup of prior cilium-health endpoint instances.
func LaunchAsEndpoint(owner endpoint.Owner, hostAddressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	var (
		cmd  = launcher.Launcher{}
		info = &models.EndpointChangeRequest{
			ContainerName: ciliumHealth,
			State:         models.EndpointStateWaitingForIdentity,
			Addressing:    &models.AddressPair{},
		}
		ip4, ip6               net.IP
		ip4Address, ip6Address string
	)

	if option.Config.EnableIPv4 {
		ip4 = node.GetIPv4HealthIP()
		info.Addressing.IPV4 = ip4.String()
		ip4WithMask := net.IPNet{IP: ip4, Mask: defaults.ContainerIPv4Mask}
		ip4Address = ip4WithMask.String()
	}

	if option.Config.EnableIPv6 {
		ip6 = node.GetIPv6HealthIP()
		info.Addressing.IPV6 = ip6.String()
		ip6WithMask := net.IPNet{IP: ip6, Mask: defaults.ContainerIPv6Mask}
		ip6Address = ip6WithMask.String()
	}

	if _, _, err := connector.SetupVethWithNames(vethName, vethPeerName, mtuConfig.GetDeviceMTU(), info); err != nil {
		return fmt.Errorf("Error while creating veth: %s", err)
	}

	pidfile := filepath.Join(option.Config.StateDir, PidfilePath)
	healthArgs := fmt.Sprintf("-d --admin=unix --passive --pidfile %s", pidfile)
	args := []string{info.ContainerName, info.InterfaceName, vethPeerName,
		ip6Address, ip4Address, ciliumHealth, healthArgs}
	prog := filepath.Join(option.Config.BpfDir, "spawn_netns.sh")
	cmd.SetTarget(prog)
	cmd.SetArgs(args)
	if err := cmd.Run(); err != nil {
		return err
	}

	// Create the endpoint
	ep, err := endpoint.NewEndpointFromChangeModel(info)
	if err != nil {
		return fmt.Errorf("Error while creating endpoint model: %s", err)
	}
	ep.SetDefaultOpts(option.Config.Opts)

	// Give the endpoint a security identity
	ep.UpdateLabels(owner, labels.LabelHealth, nil, true)

	// Wait until the cilium-health endpoint is running before setting up routes
	deadline := time.Now().Add(1 * time.Minute)
	for {
		if _, err := os.Stat(pidfile); err == nil {
			log.WithField("pidfile", pidfile).Debug("cilium-health agent running")
			break
		} else if time.Now().After(deadline) {
			return fmt.Errorf("Endpoint failed to run: %s", err)
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	// Set up the endpoint routes
	if err = configureHealthRouting(info.ContainerName, vethPeerName, hostAddressing, mtuConfig); err != nil {
		return fmt.Errorf("Error while configuring routes: %s", err)
	}

	if err := endpointmanager.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil {
		return fmt.Errorf("Error while adding endpoint: %s", err)
	}

	if !ep.SetStateLocked(endpoint.StateWaitingToRegenerate, "initial build of health endpoint") {
		endpointmanager.Remove(ep)
		return fmt.Errorf("unable to transition health endpoint to WaitingToRegenerate state")
	}

	buildSuccessful := <-ep.Regenerate(owner, &endpoint.ExternalRegenerationMetadata{
		Reason: "health daemon bootstrap",
	})
	if !buildSuccessful {
		endpointmanager.Remove(ep)
		return fmt.Errorf("unable to build health endpoint")
	}

	// Propagate health IPs to all other nodes via annotations
	if NodeEpAnnotator != nil {
		err = NodeEpAnnotator.AnnotateNode(node.GetName(), nil, nil, ip4, ip6, nil)
		if err != nil {
			return fmt.Errorf("Cannot annotate node CIDR range data: %s", err)
		}
	}

	// Initialize the health client to talk to this instance. This is why
	// the caller must limit usage of this package to a single goroutine.
	client, err = healthPkg.NewClient(fmt.Sprintf("tcp://%s:%d", ip4, healthDefaults.HTTPPathPort))
	if err != nil {
		return fmt.Errorf("Cannot establish connection to health endpoint: %s", err)
	}
	metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc()
	return nil
}
<commit_msg>health: acquire lock to set state to WaitingToRegenerate in LaunchAsEndpoint<commit_after>// Copyright 2017-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package launch

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/datapath/route"
	"github.com/cilium/cilium/pkg/defaults"
	"github.com/cilium/cilium/pkg/endpoint"
	"github.com/cilium/cilium/pkg/endpoint/connector"
	"github.com/cilium/cilium/pkg/endpointmanager"
	healthPkg "github.com/cilium/cilium/pkg/health/client"
	healthDefaults "github.com/cilium/cilium/pkg/health/defaults"
	"github.com/cilium/cilium/pkg/labels"
	"github.com/cilium/cilium/pkg/launcher"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/metrics"
	"github.com/cilium/cilium/pkg/mtu"
	"github.com/cilium/cilium/pkg/node"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/pidfile"

	"github.com/vishvananda/netlink"
)

const (
	ciliumHealth = "cilium-health"
)

var (
	// vethName is the host-side link device name for cilium-health EP.
	vethName = "cilium_health"

	// vethPeerName is the endpoint-side link device name for cilium-health.
	vethPeerName = "cilium"

	// PidfilePath
	PidfilePath = "health-endpoint.pid"

	// client is used to ping the cilium-health endpoint as a health check.
	client *healthPkg.Client

	// NodeEpAnnotator is used to annotate nodes and pods in a cluster with
	// information about this cilium-health instance.
	NodeEpAnnotator Annotator
)

func logFromCommand(cmd *exec.Cmd, netns string) error {
	scopedLog := log.WithField("netns", netns)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	go func() {
		in := bufio.NewScanner(stdout)
		for in.Scan() {
			scopedLog.Debugf("%s", in.Text())
		}
	}()
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	go func() {
		in := bufio.NewScanner(stderr)
		for in.Scan() {
			scopedLog.Infof("%s", in.Text())
		}
	}()

	return nil
}

func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	routes := []route.Route{}

	if option.Config.EnableIPv4 {
		v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU())
		if err == nil {
			routes = append(routes, v4Routes...)
		} else {
			log.Debugf("Couldn't get IPv4 routes for health routing")
		}
	}

	if option.Config.EnableIPv6 {
		v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU())
		if err != nil {
			return fmt.Errorf("Failed to get IPv6 routes")
		}
		routes = append(routes, v6Routes...)
	}

	prog := "ip"
	args := []string{"netns", "exec", netns, "bash", "-c"}
	routeCmds := []string{}
	for _, rt := range routes {
		cmd := strings.Join(rt.ToIPCommand(dev), " ")
		log.WithField("netns", netns).WithField("command", cmd).Info("Adding route")
		routeCmds = append(routeCmds, cmd)
	}
	cmd := strings.Join(routeCmds, " && ")
	args = append(args, cmd)

	log.Debugf("Running \"%s %+v\"", prog, args)
	out, err := exec.Command(prog, args...).CombinedOutput()
	if err == nil && len(out) > 0 {
		log.Warn(out)
	}

	return err
}

// PingEndpoint attempts to make an API ping request to the local cilium-health
// endpoint, and returns whether this was successful.
//
// This function must only be used from the same goroutine as LaunchAsEndpoint().
// It is safe to call PingEndpoint() before LaunchAsEndpoint() so long as the
// goroutine is the same for both calls.
func PingEndpoint() error {
	// client is shared with LaunchAsEndpoint().
	if client == nil {
		return fmt.Errorf("cilium-health endpoint hasn't yet been initialized")
	}
	_, err := client.Restapi.GetHello(nil)
	return err
}

// KillEndpoint attempts to kill any existing cilium-health endpoint if it
// exists.
//
// This is intended to be invoked in multiple situations:
// * The health endpoint has never been run before
// * The health endpoint was run during a previous run of the Cilium agent
// * The health endpoint crashed during the current run of the Cilium agent
//   and needs to be cleaned up before it is restarted.
func KillEndpoint() {
	path := filepath.Join(option.Config.StateDir, PidfilePath)
	if err := pidfile.Kill(path); err != nil {
		scopedLog := log.WithField(logfields.Path, path).WithError(err)
		scopedLog.Info("Failed to kill previous cilium-health instance")
	}
}

// CleanupEndpoint cleans up remaining resources associated with the health
// endpoint.
//
// This is expected to be called after the process is killed and the endpoint
// is removed from the endpointmanager.
func CleanupEndpoint() {
	scopedLog := log.WithField(logfields.Veth, vethName)
	if link, err := netlink.LinkByName(vethName); err == nil {
		err = netlink.LinkDel(link)
		if err != nil {
			scopedLog.WithError(err).Info("Couldn't delete cilium-health device")
		}
	} else {
		scopedLog.WithError(err).Debug("Didn't find existing device")
	}
}

// Annotator is an interface which describes anything which annotates a node
// with cilium-health metadata.
type Annotator interface {
	AnnotateNode(nodeName string, v4CIDR, v6CIDR *net.IPNet, v4HealthIP, v6HealthIP, v4CiliumHostIP net.IP) error
	AnnotatePod(k8sNamespace, k8sPodName, annotationKey, annotationValue string) error
}

// LaunchAsEndpoint launches the cilium-health agent in a nested network
// namespace and attaches it to Cilium the same way as any other endpoint,
// but with special reserved labels.
//
// CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure
// cleanup of prior cilium-health endpoint instances.
func LaunchAsEndpoint(owner endpoint.Owner, hostAddressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
	var (
		cmd  = launcher.Launcher{}
		info = &models.EndpointChangeRequest{
			ContainerName: ciliumHealth,
			State:         models.EndpointStateWaitingForIdentity,
			Addressing:    &models.AddressPair{},
		}
		ip4, ip6               net.IP
		ip4Address, ip6Address string
	)

	if option.Config.EnableIPv4 {
		ip4 = node.GetIPv4HealthIP()
		info.Addressing.IPV4 = ip4.String()
		ip4WithMask := net.IPNet{IP: ip4, Mask: defaults.ContainerIPv4Mask}
		ip4Address = ip4WithMask.String()
	}

	if option.Config.EnableIPv6 {
		ip6 = node.GetIPv6HealthIP()
		info.Addressing.IPV6 = ip6.String()
		ip6WithMask := net.IPNet{IP: ip6, Mask: defaults.ContainerIPv6Mask}
		ip6Address = ip6WithMask.String()
	}

	if _, _, err := connector.SetupVethWithNames(vethName, vethPeerName, mtuConfig.GetDeviceMTU(), info); err != nil {
		return fmt.Errorf("Error while creating veth: %s", err)
	}

	pidfile := filepath.Join(option.Config.StateDir, PidfilePath)
	healthArgs := fmt.Sprintf("-d --admin=unix --passive --pidfile %s", pidfile)
	args := []string{info.ContainerName, info.InterfaceName, vethPeerName,
		ip6Address, ip4Address, ciliumHealth, healthArgs}
	prog := filepath.Join(option.Config.BpfDir, "spawn_netns.sh")
	cmd.SetTarget(prog)
	cmd.SetArgs(args)
	if err := cmd.Run(); err != nil {
		return err
	}

	// Create the endpoint
	ep, err := endpoint.NewEndpointFromChangeModel(info)
	if err != nil {
		return fmt.Errorf("Error while creating endpoint model: %s", err)
	}
	ep.SetDefaultOpts(option.Config.Opts)

	// Give the endpoint a security identity
	ep.UpdateLabels(owner, labels.LabelHealth, nil, true)

	// Wait until the cilium-health endpoint is running before setting up routes
	deadline := time.Now().Add(1 * time.Minute)
	for {
		if _, err := os.Stat(pidfile); err == nil {
			log.WithField("pidfile", pidfile).Debug("cilium-health agent running")
			break
		} else if time.Now().After(deadline) {
			return fmt.Errorf("Endpoint failed to run: %s", err)
		} else {
			time.Sleep(1 * time.Second)
		}
	}

	// Set up the endpoint routes
	if err = configureHealthRouting(info.ContainerName, vethPeerName, hostAddressing, mtuConfig); err != nil {
		return fmt.Errorf("Error while configuring routes: %s", err)
	}

	if err := endpointmanager.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil {
		return fmt.Errorf("Error while adding endpoint: %s", err)
	}

	if err := ep.LockAlive(); err != nil {
		return err
	}
	if !ep.SetStateLocked(endpoint.StateWaitingToRegenerate, "initial build of health endpoint") {
		endpointmanager.Remove(ep)
		ep.Unlock()
		return fmt.Errorf("unable to transition health endpoint to WaitingToRegenerate state")
	}
	ep.Unlock()

	buildSuccessful := <-ep.Regenerate(owner, &endpoint.ExternalRegenerationMetadata{
		Reason: "health daemon bootstrap",
	})
	if !buildSuccessful {
		endpointmanager.Remove(ep)
		return fmt.Errorf("unable to build health endpoint")
	}

	// Propagate health IPs to all other nodes via annotations
	if NodeEpAnnotator != nil {
		err = NodeEpAnnotator.AnnotateNode(node.GetName(), nil, nil, ip4, ip6, nil)
		if err != nil {
			return fmt.Errorf("Cannot annotate node CIDR range data: %s", err)
		}
	}

	// Initialize the health client to talk to this instance. This is why
	// the caller must limit usage of this package to a single goroutine.
	client, err = healthPkg.NewClient(fmt.Sprintf("tcp://%s:%d", ip4, healthDefaults.HTTPPathPort))
	if err != nil {
		return fmt.Errorf("Cannot establish connection to health endpoint: %s", err)
	}
	metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"github.com/phzfi/RIC/server/ops"
	"github.com/phzfi/RIC/server/cache"
	"github.com/phzfi/RIC/server/images"
	"github.com/joonazan/imagick/imagick"
	"path/filepath"
	"errors"
	"fmt"
)

const (
	testfolder = "testimages/"
	resultsfolder = "testresults/"
	testgroup = "resize/"
)

const TOLERANCE = 0.0005

func TestResize(t *testing.T) {
	operator := cache.MakeOperator(512*1024*1024)
	src := ops.ImageSource{}
	src.AddRoot(testfolder + testgroup)
	blob, err := operator.GetBlob(
		src.LoadImageOp("resized.jpg"),
		ops.Resize{100, 100},
	)
	if err != nil {
		return
	}
	d, err := getDistortion(blob, "resized.jpg" )
	if err != nil {
		return
	}
	if d > TOLERANCE {
		t.Fatal(fmt.Sprintf("Bad image returned. Distortion: %v, Tolerance: %v", d, TOLERANCE))
	}
}

func getDistortion(blob images.ImageBlob, filename_cmp string) (distortion float64, err error) {
	const image_folder = testfolder + testgroup

	imagick.Initialize()
	defer imagick.Terminate()

	mw_cmp := imagick.NewMagickWand()
	defer mw_cmp.Destroy()
	err = mw_cmp.ReadImage(filepath.FromSlash(image_folder + filename_cmp))
	if err != nil {
		err = errors.New("Could not load reference image:" + err.Error())
		return
	}

	mw := imagick.NewMagickWand()
	defer mw.Destroy()
	err = mw.ReadImageBlob(blob)
	if err != nil {
		return
	}

	// Save image, just in case someone wants to look at it
	err = mw.WriteImage(filepath.FromSlash(resultsfolder + testgroup + filename_cmp))
	if err != nil {
		return
	}

	trash, distortion := mw.CompareImages(mw_cmp, imagick.METRIC_MEAN_SQUARED_ERROR)
	trash.Destroy()

	return
}
<commit_msg>fixed test to use toresize instead of resized; editor ran go fmt<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/joonazan/imagick/imagick"
	"github.com/phzfi/RIC/server/cache"
	"github.com/phzfi/RIC/server/images"
	"github.com/phzfi/RIC/server/ops"
	"path/filepath"
	"testing"
)

const (
	testfolder    = "testimages/"
	resultsfolder = "testresults/"
	testgroup     = "resize/"
)

const TOLERANCE = 0.002

func TestResize(t *testing.T) {
	operator := cache.MakeOperator(512 * 1024 * 1024)
	src := ops.ImageSource{}
	src.AddRoot(testfolder + testgroup)
	blob, err := operator.GetBlob(
		src.LoadImageOp("toresize.jpg"),
		ops.Resize{100, 100},
	)
	if err != nil {
		return
	}
	d, err := getDistortion(blob, "resized.jpg")
	if err != nil {
		return
	}
	if d > TOLERANCE {
		t.Fatal(fmt.Sprintf("Bad image returned. Distortion: %v, Tolerance: %v", d, TOLERANCE))
	}
}

func getDistortion(blob images.ImageBlob, filename_cmp string) (distortion float64, err error) {
	const image_folder = testfolder + testgroup

	imagick.Initialize()
	defer imagick.Terminate()

	mw_cmp := imagick.NewMagickWand()
	defer mw_cmp.Destroy()
	err = mw_cmp.ReadImage(filepath.FromSlash(image_folder + filename_cmp))
	if err != nil {
		err = errors.New("Could not load reference image:" + err.Error())
		return
	}

	mw := imagick.NewMagickWand()
	defer mw.Destroy()
	err = mw.ReadImageBlob(blob)
	if err != nil {
		return
	}

	// Save image, just in case someone wants to look at it
	err = mw.WriteImage(filepath.FromSlash(resultsfolder + testgroup + filename_cmp))
	if err != nil {
		return
	}

	trash, distortion := mw.CompareImages(mw_cmp, imagick.METRIC_MEAN_SQUARED_ERROR)
	trash.Destroy()

	return
}
<|endoftext|>


<commit_before><commit_msg>Add wait time to make the object populated by controller (#26988)<commit_after><|endoftext|>