<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * rpc/http_service.go                                    *
 *                                                        *
 * hprose http service for Go.                            *
 *                                                        *
 * LastModified: Sep 12, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package rpc

import (
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"
)

// HTTPContext is the hprose http context
type HTTPContext struct {
	*ServiceContext
	Response http.ResponseWriter
	Request  *http.Request
}

// HTTPService is the hprose http service
type HTTPService struct {
	*BaseService
	P3P                          bool
	GET                          bool
	CrossDomain                  bool
	accessControlAllowOrigins    map[string]bool
	lastModified                 string
	etag                         string
	crossDomainXMLFile           string
	crossDomainXMLContent        []byte
	clientAccessPolicyXMLFile    string
	clientAccessPolicyXMLContent []byte
}

type sendHeaderEvent interface {
	OnSendHeader(context Context)
}

type sendHeaderEvent2 interface {
	OnSendHeader(context *HTTPContext)
}

type httpFixer struct{}

func (httpFixer) FixArguments(args []reflect.Value, context *ServiceContext) {
	i := len(args) - 1
	lastParamType := args[i].Type()
	typ := (*emptyInterface)(unsafe.Pointer(&lastParamType)).ptr
	if typ == httpContextType {
		if c, ok := context.TransportContext.(*HTTPContext); ok {
			args[i] = reflect.ValueOf(c)
		}
		return
	}
	if typ == httpRequestType {
		if c, ok := context.TransportContext.(*HTTPContext); ok {
			args[i] = reflect.ValueOf(c.Request)
		}
		return
	}
	fixArguments(args, context)
}

// NewHTTPService is the constructor of HTTPService
func NewHTTPService() (service *HTTPService) {
	t := time.Now().UTC()
	rand.Seed(t.UnixNano())
	service = new(HTTPService)
	service.BaseService = NewBaseService()
	service.P3P = true
	service.GET = true
	service.CrossDomain = true
	service.accessControlAllowOrigins = make(map[string]bool)
	service.lastModified = t.Format(time.RFC1123)
	service.etag = `"` + strconv.FormatInt(rand.Int63(), 16) + `"`
	service.fixer = httpFixer{}
	return
}

func (service *HTTPService) crossDomainXMLHandler(
	response http.ResponseWriter, request *http.Request) bool {
	if service.crossDomainXMLContent == nil ||
		strings.ToLower(request.URL.Path) != "/crossdomain.xml" {
		return false
	}
	if request.Header.Get("if-modified-since") == service.lastModified &&
		request.Header.Get("if-none-match") == service.etag {
		response.WriteHeader(304)
	} else {
		contentLength := len(service.crossDomainXMLContent)
		header := response.Header()
		header.Set("Last-Modified", service.lastModified)
		header.Set("Etag", service.etag)
		header.Set("Content-Type", "text/xml")
		header.Set("Content-Length", strconv.Itoa(contentLength))
		response.Write(service.crossDomainXMLContent)
	}
	return true
}

func (service *HTTPService) clientAccessPolicyXMLHandler(
	response http.ResponseWriter, request *http.Request) bool {
	if service.clientAccessPolicyXMLContent == nil ||
		strings.ToLower(request.URL.Path) != "/clientaccesspolicy.xml" {
		return false
	}
	if request.Header.Get("if-modified-since") == service.lastModified &&
		request.Header.Get("if-none-match") == service.etag {
		response.WriteHeader(304)
	} else {
		contentLength := len(service.clientAccessPolicyXMLContent)
		header := response.Header()
		header.Set("Last-Modified", service.lastModified)
		header.Set("Etag", service.etag)
		header.Set("Content-Type", "text/xml")
		header.Set("Content-Length", strconv.Itoa(contentLength))
		response.Write(service.clientAccessPolicyXMLContent)
	}
	return true
}

func (service *HTTPService) sendHeader(context *HTTPContext) {
	switch event := service.Event.(type) {
	case sendHeaderEvent:
		event.OnSendHeader(context)
	case sendHeaderEvent2:
		event.OnSendHeader(context)
	}
	header := context.Response.Header()
	header.Set("Content-Type", "text/plain")
	if service.P3P {
		header.Set("P3P", `CP="CAO DSP COR CUR ADM DEV TAI PSA PSD IVAi IVDi `+
			`CONi TELo OTPi OUR DELi SAMi OTRi UNRi PUBi IND PHY ONL `+
			`UNI PUR FIN COM NAV INT DEM CNT STA POL HEA PRE GOV"`)
	}
	if service.CrossDomain {
		origin := context.Request.Header.Get("origin")
		if origin != "" && origin != "null" {
			if len(service.accessControlAllowOrigins) == 0 || service.accessControlAllowOrigins[origin] {
				header.Set("Access-Control-Allow-Origin", origin)
				header.Set("Access-Control-Allow-Credentials", "true")
			}
		} else {
			header.Set("Access-Control-Allow-Origin", "*")
		}
	}
}

// AddAccessControlAllowOrigin add access control allow origin
func (service *HTTPService) AddAccessControlAllowOrigin(origins ...string) {
	for _, origin := range origins {
		service.accessControlAllowOrigins[origin] = true
	}
}

// RemoveAccessControlAllowOrigin remove access control allow origin
func (service *HTTPService) RemoveAccessControlAllowOrigin(origins ...string) {
	for _, origin := range origins {
		delete(service.accessControlAllowOrigins, origin)
	}
}

// CrossDomainXMLFile return the cross domain xml file
func (service *HTTPService) CrossDomainXMLFile() string {
	return service.crossDomainXMLFile
}

// CrossDomainXMLContent return the cross domain xml content
func (service *HTTPService) CrossDomainXMLContent() []byte {
	return service.crossDomainXMLContent
}

// ClientAccessPolicyXMLFile return the client access policy xml file
func (service *HTTPService) ClientAccessPolicyXMLFile() string {
	return service.clientAccessPolicyXMLFile
}

// ClientAccessPolicyXMLContent return the client access policy xml content
func (service *HTTPService) ClientAccessPolicyXMLContent() []byte {
	return service.clientAccessPolicyXMLContent
}

// SetCrossDomainXMLFile set the cross domain xml file
func (service *HTTPService) SetCrossDomainXMLFile(filename string) {
	service.crossDomainXMLFile = filename
	service.crossDomainXMLContent, _ = ioutil.ReadFile(filename)
}

// SetClientAccessPolicyXMLFile set the client access policy xml file
func (service *HTTPService) SetClientAccessPolicyXMLFile(filename string) {
	service.clientAccessPolicyXMLFile = filename
	service.clientAccessPolicyXMLContent, _ = ioutil.ReadFile(filename)
}

// SetCrossDomainXMLContent set the cross domain xml content
func (service *HTTPService) SetCrossDomainXMLContent(content []byte) {
	service.crossDomainXMLFile = ""
	service.crossDomainXMLContent = content
}

// SetClientAccessPolicyXMLContent set the client access policy xml content
func (service *HTTPService) SetClientAccessPolicyXMLContent(content []byte) {
	service.clientAccessPolicyXMLFile = ""
	service.clientAccessPolicyXMLContent = content
}

func (service *HTTPService) readAll(request *http.Request) ([]byte, error) {
	if request.ContentLength > 0 {
		data := make([]byte, request.ContentLength)
		_, err := io.ReadFull(request.Body, data)
		return data, err
	}
	if request.ContentLength < 0 {
		return ioutil.ReadAll(request.Body)
	}
	return make([]byte, 0), nil
}

// Serve ...
func (service *HTTPService) Serve(
	response http.ResponseWriter, request *http.Request,
	userData map[string]interface{}) {
	if service.clientAccessPolicyXMLHandler(response, request) ||
		service.crossDomainXMLHandler(response, request) {
		return
	}
	context := new(HTTPContext)
	context.ServiceContext = NewServiceContext(nil)
	context.ServiceContext.TransportContext = context
	context.Response = response
	context.Request = request
	if userData != nil {
		for k, v := range userData {
			context.SetInterface(k, v)
		}
	}
	service.sendHeader(context)
	switch request.Method {
	case "GET":
		if service.GET {
			response.Write(service.doFunctionList(context.ServiceContext))
		} else {
			response.WriteHeader(403)
		}
	case "POST":
		data, err := service.readAll(request)
		request.Body.Close()
		if err != nil {
			response.Write(service.endError(err, context))
		}
		resp, err := service.Handle(data, context.ServiceContext).Get()
		if err != nil {
			response.Write(service.endError(err, context))
		} else {
			response.Write(resp.([]byte))
		}
	}
}

// ServeHTTP is the hprose http handler method
func (service *HTTPService) ServeHTTP(
	response http.ResponseWriter, request *http.Request) {
	service.Serve(response, request, nil)
}
<commit_msg>Update http_service.go<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * rpc/http_service.go                                    *
 *                                                        *
 * hprose http service for Go.                            *
 *                                                        *
 * LastModified: Sep 12, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package rpc

import (
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unsafe"

	"github.com/hprose/hprose-golang/pool"
)

// HTTPContext is the hprose http context
type HTTPContext struct {
	*ServiceContext
	Response http.ResponseWriter
	Request  *http.Request
}

// HTTPService is the hprose http service
type HTTPService struct {
	*BaseService
	P3P                          bool
	GET                          bool
	CrossDomain                  bool
	accessControlAllowOrigins    map[string]bool
	lastModified                 string
	etag                         string
	crossDomainXMLFile           string
	crossDomainXMLContent        []byte
	clientAccessPolicyXMLFile    string
	clientAccessPolicyXMLContent []byte
}

type sendHeaderEvent interface {
	OnSendHeader(context Context)
}

type sendHeaderEvent2 interface {
	OnSendHeader(context *HTTPContext)
}

type httpFixer struct{}

func (httpFixer) FixArguments(args []reflect.Value, context *ServiceContext) {
	i := len(args) - 1
	lastParamType := args[i].Type()
	typ := (*emptyInterface)(unsafe.Pointer(&lastParamType)).ptr
	if typ == httpContextType {
		if c, ok := context.TransportContext.(*HTTPContext); ok {
			args[i] = reflect.ValueOf(c)
		}
		return
	}
	if typ == httpRequestType {
		if c, ok := context.TransportContext.(*HTTPContext); ok {
			args[i] = reflect.ValueOf(c.Request)
		}
		return
	}
	fixArguments(args, context)
}

// NewHTTPService is the constructor of HTTPService
func NewHTTPService() (service *HTTPService) {
	t := time.Now().UTC()
	rand.Seed(t.UnixNano())
	service = new(HTTPService)
	service.BaseService = NewBaseService()
	service.P3P = true
	service.GET = true
	service.CrossDomain = true
	service.accessControlAllowOrigins = make(map[string]bool)
	service.lastModified = t.Format(time.RFC1123)
	service.etag = `"` + strconv.FormatInt(rand.Int63(), 16) + `"`
	service.fixer = httpFixer{}
	return
}

func (service *HTTPService) crossDomainXMLHandler(
	response http.ResponseWriter, request *http.Request) bool {
	if service.crossDomainXMLContent == nil ||
		strings.ToLower(request.URL.Path) != "/crossdomain.xml" {
		return false
	}
	if request.Header.Get("if-modified-since") == service.lastModified &&
		request.Header.Get("if-none-match") == service.etag {
		response.WriteHeader(304)
	} else {
		contentLength := len(service.crossDomainXMLContent)
		header := response.Header()
		header.Set("Last-Modified", service.lastModified)
		header.Set("Etag", service.etag)
		header.Set("Content-Type", "text/xml")
		header.Set("Content-Length", strconv.Itoa(contentLength))
		response.Write(service.crossDomainXMLContent)
	}
	return true
}

func (service *HTTPService) clientAccessPolicyXMLHandler(
	response http.ResponseWriter, request *http.Request) bool {
	if service.clientAccessPolicyXMLContent == nil ||
		strings.ToLower(request.URL.Path) != "/clientaccesspolicy.xml" {
		return false
	}
	if request.Header.Get("if-modified-since") == service.lastModified &&
		request.Header.Get("if-none-match") == service.etag {
		response.WriteHeader(304)
	} else {
		contentLength := len(service.clientAccessPolicyXMLContent)
		header := response.Header()
		header.Set("Last-Modified", service.lastModified)
		header.Set("Etag", service.etag)
		header.Set("Content-Type", "text/xml")
		header.Set("Content-Length", strconv.Itoa(contentLength))
		response.Write(service.clientAccessPolicyXMLContent)
	}
	return true
}

func (service *HTTPService) sendHeader(context *HTTPContext) {
	switch event := service.Event.(type) {
	case sendHeaderEvent:
		event.OnSendHeader(context)
	case sendHeaderEvent2:
		event.OnSendHeader(context)
	}
	header := context.Response.Header()
	header.Set("Content-Type", "text/plain")
	if service.P3P {
		header.Set("P3P", `CP="CAO DSP COR CUR ADM DEV TAI PSA PSD IVAi IVDi `+
			`CONi TELo OTPi OUR DELi SAMi OTRi UNRi PUBi IND PHY ONL `+
			`UNI PUR FIN COM NAV INT DEM CNT STA POL HEA PRE GOV"`)
	}
	if service.CrossDomain {
		origin := context.Request.Header.Get("origin")
		if origin != "" && origin != "null" {
			if len(service.accessControlAllowOrigins) == 0 || service.accessControlAllowOrigins[origin] {
				header.Set("Access-Control-Allow-Origin", origin)
				header.Set("Access-Control-Allow-Credentials", "true")
			}
		} else {
			header.Set("Access-Control-Allow-Origin", "*")
		}
	}
}

// AddAccessControlAllowOrigin add access control allow origin
func (service *HTTPService) AddAccessControlAllowOrigin(origins ...string) {
	for _, origin := range origins {
		service.accessControlAllowOrigins[origin] = true
	}
}

// RemoveAccessControlAllowOrigin remove access control allow origin
func (service *HTTPService) RemoveAccessControlAllowOrigin(origins ...string) {
	for _, origin := range origins {
		delete(service.accessControlAllowOrigins, origin)
	}
}

// CrossDomainXMLFile return the cross domain xml file
func (service *HTTPService) CrossDomainXMLFile() string {
	return service.crossDomainXMLFile
}

// CrossDomainXMLContent return the cross domain xml content
func (service *HTTPService) CrossDomainXMLContent() []byte {
	return service.crossDomainXMLContent
}

// ClientAccessPolicyXMLFile return the client access policy xml file
func (service *HTTPService) ClientAccessPolicyXMLFile() string {
	return service.clientAccessPolicyXMLFile
}

// ClientAccessPolicyXMLContent return the client access policy xml content
func (service *HTTPService) ClientAccessPolicyXMLContent() []byte {
	return service.clientAccessPolicyXMLContent
}

// SetCrossDomainXMLFile set the cross domain xml file
func (service *HTTPService) SetCrossDomainXMLFile(filename string) {
	service.crossDomainXMLFile = filename
	service.crossDomainXMLContent, _ = ioutil.ReadFile(filename)
}

// SetClientAccessPolicyXMLFile set the client access policy xml file
func (service *HTTPService) SetClientAccessPolicyXMLFile(filename string) {
	service.clientAccessPolicyXMLFile = filename
	service.clientAccessPolicyXMLContent, _ = ioutil.ReadFile(filename)
}

// SetCrossDomainXMLContent set the cross domain xml content
func (service *HTTPService) SetCrossDomainXMLContent(content []byte) {
	service.crossDomainXMLFile = ""
	service.crossDomainXMLContent = content
}

// SetClientAccessPolicyXMLContent set the client access policy xml content
func (service *HTTPService) SetClientAccessPolicyXMLContent(content []byte) {
	service.clientAccessPolicyXMLFile = ""
	service.clientAccessPolicyXMLContent = content
}

func (service *HTTPService) readAll(request *http.Request) ([]byte, error) {
	if request.ContentLength > 0 {
		data := make([]byte, request.ContentLength)
		_, err := io.ReadFull(request.Body, data)
		return data, err
	}
	if request.ContentLength < 0 {
		return ioutil.ReadAll(request.Body)
	}
	return make([]byte, 0), nil
}

// Serve ...
func (service *HTTPService) Serve(
	response http.ResponseWriter, request *http.Request,
	userData map[string]interface{}) {
	if service.clientAccessPolicyXMLHandler(response, request) ||
		service.crossDomainXMLHandler(response, request) {
		return
	}
	context := new(HTTPContext)
	context.ServiceContext = NewServiceContext(nil)
	context.ServiceContext.TransportContext = context
	context.Response = response
	context.Request = request
	if userData != nil {
		for k, v := range userData {
			context.SetInterface(k, v)
		}
	}
	service.sendHeader(context)
	switch request.Method {
	case "GET":
		if service.GET {
			response.Write(service.doFunctionList(context.ServiceContext))
		} else {
			response.WriteHeader(403)
		}
	case "POST":
		data, err := service.readAll(request)
		request.Body.Close()
		if err != nil {
			response.Write(service.endError(err, context))
		}
		resp, err := service.Handle(data, context.ServiceContext).Get()
		if err != nil {
			response.Write(service.endError(err, context))
		} else if data, ok := resp.([]byte); ok {
			response.Write(data)
			pool.Recycle(data)
		} else {
			response.WriteHeader(500)
		}
	}
}

// ServeHTTP is the hprose http handler method
func (service *HTTPService) ServeHTTP(
	response http.ResponseWriter, request *http.Request) {
	service.Serve(response, request, nil)
}
<|endoftext|>


<commit_before>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	// gif support
	_ "image/gif"
	// jpeg support
	_ "image/jpeg"
	// png support
	_ "image/png"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	shortid "github.com/ventu-io/go-shortid"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"

	local "github.com/eirka/eirka-post/config"
)

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

func init() {

	var err error

	// test for ImageMagick
	_, err = exec.Command("convert", "--version").Output()
	if err != nil {
		panic("ImageMagick not found")
	}

}

// FileUploader defines the file processing functions
type FileUploader interface {
	// struct integrity
	IsValid() bool
	IsValidPost() bool

	// image processing
	SaveImage() (err error)
	checkReqExt() (err error)
	copyBytes() (err error)
	getHash() (err error)
	checkBanned() (err error)
	checkDuplicate() (err error)
	checkMagic() (err error)
	getStats() (err error)
	saveFile() (err error)
	makeFilenames()
	createThumbnail(maxwidth, maxheight int) (err error)
	copyToS3() (err error)

	// webm specific functions
	checkWebM() (err error)
	createWebMThumbnail() (err error)

	// avatar functions
	SaveAvatar() (err error)
	avatarToS3() (err error)
}

// ImageType defines an image and its metadata for processing
type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Ib          uint
	Filename    string
	Thumbnail   string
	Filepath    string
	Thumbpath   string
	Ext         string
	MD5         string
	SHA         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       *bytes.Buffer
	mime        string
	duration    int
	video       bool
	avatar      bool
}

var _ = FileUploader(&ImageType{})

// IsValid will check struct integrity
func (i *ImageType) IsValid() bool {

	if i.Filename == "" {
		return false
	}

	if i.Filepath == "" {
		return false
	}

	if i.Thumbnail == "" {
		return false
	}

	if i.Thumbpath == "" {
		return false
	}

	if i.Ib == 0 {
		return false
	}

	if i.Ext == "" {
		return false
	}

	if i.MD5 == "" {
		return false
	}

	if i.SHA == "" {
		return false
	}

	if i.mime == "" {
		return false
	}

	return true
}

// IsValidPost will check final struct integrity
func (i *ImageType) IsValidPost() bool {
	if i.OrigWidth == 0 {
		return false
	}

	if i.OrigHeight == 0 {
		return false
	}

	if i.ThumbWidth == 0 {
		return false
	}

	if i.ThumbHeight == 0 {
		return false
	}

	return true
}

// SaveImage runs the entire file processing pipeline
func (i *ImageType) SaveImage() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// copy the multipart file into a buffer
	err = i.copyBytes()
	if err != nil {
		return
	}

	// get file md5
	err = i.getHash()
	if err != nil {
		return
	}

	// check to see if the file is banned
	err = i.checkBanned()
	if err != nil {
		return
	}

	// check to see if the file already exists
	err = i.checkDuplicate()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// process a webm
	if i.video {
		// check the webm info
		err = i.checkWebM()
		if err != nil {
			return
		}

		// create thumbnail from webm
		err = i.createWebMThumbnail()
		if err != nil {
			return
		}

	}

	// create a thumbnail
	err = i.createThumbnail(config.Settings.Limits.ThumbnailMaxWidth, config.Settings.Limits.ThumbnailMaxHeight)
	if err != nil {
		return
	}

	// copy the file to s3
	err = i.copyToS3()
	if err != nil {
		return
	}

	// check final state
	if !i.IsValidPost() {
		return errors.New("ImageType is not valid")
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("No file extension")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(ext) {
		return errors.New("Format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {
	return validExt[strings.ToLower(ext)]
}

// Copy the multipart file into a bytes buffer
func (i *ImageType) copyBytes() (err error) {
	defer i.File.Close()

	i.image = new(bytes.Buffer)

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, i.File)
	if err != nil {
		return errors.New("Problem copying file to buffer")
	}

	return
}

// Get image MD5 and write file into buffer
func (i *ImageType) getHash() (err error) {

	hasher := md5.New()

	sha := sha1.New()

	// read into hasher for md5
	_, err = io.Copy(hasher, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating MD5 hash")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	// read into hasher for md5
	_, err = io.Copy(sha, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating SHA1 hash")
	}

	// Set sha1
	i.SHA = hex.EncodeToString(sha.Sum(nil))

	return

}

// check if the md5 is a banned file
func (i *ImageType) checkBanned() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on file banned check")
	}

	var check bool

	err = dbase.QueryRow(`SELECT count(*) FROM banned_files WHERE ban_hash = ?`, i.MD5).Scan(&check)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("File is banned")
	}

	return
}

// check if the md5 is already in the database
func (i *ImageType) checkDuplicate() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on duplicate check")
	}

	if i.Ib == 0 {
		return errors.New("No imageboard set on duplicate check")
	}

	var check bool
	var thread, post sql.NullInt64

	err = dbase.QueryRow(`select count(1),posts.post_num,threads.thread_id from threads
	LEFT JOIN posts on threads.thread_id = posts.thread_id
	LEFT JOIN images on posts.post_id = images.post_id
	WHERE image_hash = ? AND ib_id = ? AND post_deleted != 1`, i.MD5, i.Ib).Scan(&check, &post, &thread)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("Image has already been posted. Thread: %d Post: %d", thread.Int64, post.Int64)
	}

	return
}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
		i.video = true
	default:
		return errors.New("Unknown file type")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(i.Ext) {
		return errors.New("Format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// skip if its a video since we cant decode it
	if i.video {
		return
	}

	// decode image config
	img, _, err := image.DecodeConfig(bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return fmt.Errorf("Image width too large. Max: %dpx", config.Settings.Limits.ImageMaxWidth)
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return fmt.Errorf("Image width too small. Min: %dpx", config.Settings.Limits.ImageMinWidth)
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return fmt.Errorf("Image height too large. Max: %dpx", config.Settings.Limits.ImageMaxHeight)
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return fmt.Errorf("Image height too small. Min: %dpx", config.Settings.Limits.ImageMinHeight)
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return fmt.Errorf("Image filesize too large. Max: %dMB", (config.Settings.Limits.ImageMaxSize/1024)/1024)
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	defer i.image.Reset()

	i.makeFilenames()

	// avatar filename is the users id
	if i.avatar {
		i.Thumbnail = fmt.Sprintf("%d.png", i.Ib)
		i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)
	}

	if !i.IsValid() {
		return errors.New("ImageType is not valid")
	}

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("Problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem saving file")
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// get new short id generator
	sid := shortid.MustNew(1, shortid.DefaultABC, 9001)

	// generate filename
	filename := sid.MustGenerate()

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", filename, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", filename)

	// set the full file path
	i.Filepath = filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	// set the full thumbnail path
	i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)

	return

}

func (i *ImageType) createThumbnail(maxwidth, maxheight int) (err error) {

	var imagef string

	if i.video {
		imagef = fmt.Sprintf("%s[0]", i.Thumbpath)
	} else {
		imagef = fmt.Sprintf("%s[0]", i.Filepath)
	}

	originalDimensions := fmt.Sprintf("%dx%d", i.OrigWidth, i.OrigHeight)

	var args []string

	// different options for avatars
	if i.avatar {
		args = []string{
			"-size",
			originalDimensions,
			imagef,
			"-background",
			"none",
			"-thumbnail",
			fmt.Sprintf("%dx%d^", maxwidth, maxheight),
			"-gravity",
			"center",
			"-extent",
			fmt.Sprintf("%dx%d", maxwidth, maxheight),
			i.Thumbpath,
		}
	} else {
		args = []string{
			"-background",
			"white",
			"-flatten",
			"-size",
			originalDimensions,
			"-resize",
			fmt.Sprintf("%dx%d>", maxwidth, maxheight),
			"-quality",
			"90",
			imagef,
			i.Thumbpath,
		}
	}

	_, err = exec.Command("convert", args...).Output()
	if err != nil {
		return errors.New("Problem making thumbnail")
	}

	thumb, err := os.Open(i.Thumbpath)
	if err != nil {
		return errors.New("Problem creating thumbnail file")
	}
	defer thumb.Close()

	img, _, err := image.DecodeConfig(thumb)
	if err != nil {
		return errors.New("Problem decoding thumbnail")
	}

	i.ThumbWidth = img.Width
	i.ThumbHeight = img.Height

	return

}

func (i *ImageType) copyToS3() (err error) {

	s3 := amazon.New()

	err = s3.Save(i.Filepath, fmt.Sprintf("src/%s", i.Filename), i.mime, false)
	if err != nil {
		return
	}

	err = s3.Save(i.Thumbpath, fmt.Sprintf("thumb/%s", i.Thumbnail), "image/jpeg", false)
	if err != nil {
		return
	}

	return
}
<commit_msg>image duplicate checker shouldnt check for deleted files<commit_after>package utils

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"fmt"
	"image"
	// gif support
	_ "image/gif"
	// jpeg support
	_ "image/jpeg"
	// png support
	_ "image/png"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	shortid "github.com/ventu-io/go-shortid"

	"github.com/eirka/eirka-libs/amazon"
	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"

	local "github.com/eirka/eirka-post/config"
)

// valid file extensions
var validExt = map[string]bool{
	".jpg":  true,
	".jpeg": true,
	".png":  true,
	".gif":  true,
	".webm": true,
}

func init() {

	var err error

	// test for ImageMagick
	_, err = exec.Command("convert", "--version").Output()
	if err != nil {
		panic("ImageMagick not found")
	}

}

// FileUploader defines the file processing functions
type FileUploader interface {
	// struct integrity
	IsValid() bool
	IsValidPost() bool

	// image processing
	SaveImage() (err error)
	checkReqExt() (err error)
	copyBytes() (err error)
	getHash() (err error)
	checkBanned() (err error)
	checkDuplicate() (err error)
	checkMagic() (err error)
	getStats() (err error)
	saveFile() (err error)
	makeFilenames()
	createThumbnail(maxwidth, maxheight int) (err error)
	copyToS3() (err error)

	// webm specific functions
	checkWebM() (err error)
	createWebMThumbnail() (err error)

	// avatar functions
	SaveAvatar() (err error)
	avatarToS3() (err error)
}

// ImageType defines an image and its metadata for processing
type ImageType struct {
	File        multipart.File
	Header      *multipart.FileHeader
	Ib          uint
	Filename    string
	Thumbnail   string
	Filepath    string
	Thumbpath   string
	Ext         string
	MD5         string
	SHA         string
	OrigWidth   int
	OrigHeight  int
	ThumbWidth  int
	ThumbHeight int
	image       *bytes.Buffer
	mime        string
	duration    int
	video       bool
	avatar      bool
}

var _ = FileUploader(&ImageType{})

// IsValid will check struct integrity
func (i *ImageType) IsValid() bool {

	if i.Filename == "" {
		return false
	}

	if i.Filepath == "" {
		return false
	}

	if i.Thumbnail == "" {
		return false
	}

	if i.Thumbpath == "" {
		return false
	}

	if i.Ib == 0 {
		return false
	}

	if i.Ext == "" {
		return false
	}

	if i.MD5 == "" {
		return false
	}

	if i.SHA == "" {
		return false
	}

	if i.mime == "" {
		return false
	}

	return true
}

// IsValidPost will check final struct integrity
func (i *ImageType) IsValidPost() bool {
	if i.OrigWidth == 0 {
		return false
	}

	if i.OrigHeight == 0 {
		return false
	}

	if i.ThumbWidth == 0 {
		return false
	}

	if i.ThumbHeight == 0 {
		return false
	}

	return true
}

// SaveImage runs the entire file processing pipeline
func (i *ImageType) SaveImage() (err error) {

	// check given file ext
	err = i.checkReqExt()
	if err != nil {
		return
	}

	// copy the multipart file into a buffer
	err = i.copyBytes()
	if err != nil {
		return
	}

	// get file md5
	err = i.getHash()
	if err != nil {
		return
	}

	// check to see if the file is banned
	err = i.checkBanned()
	if err != nil {
		return
	}

	// check to see if the file already exists
	err = i.checkDuplicate()
	if err != nil {
		return
	}

	// check file magic sig
	err = i.checkMagic()
	if err != nil {
		return
	}

	// check image stats
	err = i.getStats()
	if err != nil {
		return
	}

	// save the file to disk
	err = i.saveFile()
	if err != nil {
		return
	}

	// process a webm
	if i.video {
		// check the webm info
		err = i.checkWebM()
		if err != nil {
			return
		}

		// create thumbnail from webm
		err = i.createWebMThumbnail()
		if err != nil {
			return
		}

	}

	// create a thumbnail
	err = i.createThumbnail(config.Settings.Limits.ThumbnailMaxWidth, config.Settings.Limits.ThumbnailMaxHeight)
	if err != nil {
		return
	}

	// copy the file to s3
	err = i.copyToS3()
	if err != nil {
		return
	}

	// check final state
	if !i.IsValidPost() {
		return errors.New("ImageType is not valid")
	}

	return

}

// Get file extension from request header
func (i *ImageType) checkReqExt() (err error) {
	// Get ext from request header
	name := i.Header.Filename
	ext := filepath.Ext(name)

	if ext == "" {
		return errors.New("No file extension")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(ext) {
		return errors.New("Format not supported")
	}

	return

}

// Check if file ext allowed
func isAllowedExt(ext string) bool {
	return validExt[strings.ToLower(ext)]
}

// Copy the multipart file into a bytes buffer
func (i *ImageType) copyBytes() (err error) {
	defer i.File.Close()

	i.image = new(bytes.Buffer)

	// Save file and also read into hasher for md5
	_, err = io.Copy(i.image, i.File)
	if err != nil {
		return errors.New("Problem copying file to buffer")
	}

	return
}

// Get image MD5 and write file into buffer
func (i *ImageType) getHash() (err error) {

	hasher := md5.New()

	sha := sha1.New()

	// read into hasher for md5
	_, err = io.Copy(hasher, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating MD5 hash")
	}

	// Set md5sum from hasher
	i.MD5 = hex.EncodeToString(hasher.Sum(nil))

	// read into hasher for md5
	_, err = io.Copy(sha, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem creating SHA1 hash")
	}

	// Set sha1
	i.SHA = hex.EncodeToString(sha.Sum(nil))

	return

}

// check if the md5 is a banned file
func (i *ImageType) checkBanned() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on file banned check")
	}

	var check bool

	err = dbase.QueryRow(`SELECT count(*) FROM banned_files WHERE ban_hash = ?`, i.MD5).Scan(&check)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("File is banned")
	}

	return
}

// check if the md5 is already in the database
func (i *ImageType) checkDuplicate() (err error) {

	// Get Database handle
	dbase, err := db.GetDb()
	if err != nil {
		return
	}

	if i.MD5 == "" {
		return errors.New("No hash set on duplicate check")
	}

	if i.Ib == 0 {
		return errors.New("No imageboard set on duplicate check")
	}

	var check bool
	var thread, post sql.NullInt64

	err = dbase.QueryRow(`select count(1),posts.post_num,threads.thread_id from threads
	LEFT JOIN posts on threads.thread_id = posts.thread_id
	LEFT JOIN images on posts.post_id = images.post_id
	WHERE image_hash = ? AND ib_id = ? AND post_deleted = 0`, i.MD5, i.Ib).Scan(&check, &post, &thread)
	if err != nil {
		return
	}

	// return error if it exists
	if check {
		return fmt.Errorf("Image has already been posted. Thread: %d Post: %d", thread.Int64, post.Int64)
	}

	return
}

func (i *ImageType) checkMagic() (err error) {

	// detect the mime type
	i.mime = http.DetectContentType(i.image.Bytes())

	switch i.mime {
	case "image/png":
		i.Ext = ".png"
	case "image/jpeg":
		i.Ext = ".jpg"
	case "image/gif":
		i.Ext = ".gif"
	case "video/webm":
		i.Ext = ".webm"
		i.video = true
	default:
		return errors.New("Unknown file type")
	}

	// Check to see if extension is allowed
	if !isAllowedExt(i.Ext) {
		return errors.New("Format not supported")
	}

	return

}

func (i *ImageType) getStats() (err error) {

	// skip if its a video since we cant decode it
	if i.video {
		return
	}

	// decode image config
	img, _, err := image.DecodeConfig(bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem decoding image")
	}

	// set original width
	i.OrigWidth = img.Width
	// set original height
	i.OrigHeight = img.Height

	// Check against maximum sizes
	switch {
	case i.OrigWidth > config.Settings.Limits.ImageMaxWidth:
		return fmt.Errorf("Image width too large. Max: %dpx", config.Settings.Limits.ImageMaxWidth)
	case img.Width < config.Settings.Limits.ImageMinWidth:
		return fmt.Errorf("Image width too small. Min: %dpx", config.Settings.Limits.ImageMinWidth)
	case i.OrigHeight > config.Settings.Limits.ImageMaxHeight:
		return fmt.Errorf("Image height too large. Max: %dpx", config.Settings.Limits.ImageMaxHeight)
	case img.Height < config.Settings.Limits.ImageMinHeight:
		return fmt.Errorf("Image height too small. Min: %dpx", config.Settings.Limits.ImageMinHeight)
	case i.image.Len() > config.Settings.Limits.ImageMaxSize:
		return fmt.Errorf("Image filesize too large. Max: %dMB", (config.Settings.Limits.ImageMaxSize/1024)/1024)
	}

	return

}

func (i *ImageType) saveFile() (err error) {

	defer i.image.Reset()

	i.makeFilenames()

	// avatar filename is the users id
	if i.avatar {
		i.Thumbnail = fmt.Sprintf("%d.png", i.Ib)
		i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)
	}

	if !i.IsValid() {
		return errors.New("ImageType is not valid")
	}

	imagefile := filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	image, err := os.Create(imagefile)
	if err != nil {
		return errors.New("Problem saving file")
	}
	defer image.Close()

	_, err = io.Copy(image, bytes.NewReader(i.image.Bytes()))
	if err != nil {
		return errors.New("Problem saving file")
	}

	return

}

// Make a random unix time filename
func (i *ImageType) makeFilenames() {

	// get new short id generator
	sid := shortid.MustNew(1, shortid.DefaultABC, 9001)

	// generate filename
	filename := sid.MustGenerate()

	// Append ext to filename
	i.Filename = fmt.Sprintf("%s%s", filename, i.Ext)

	// Append jpg to thumbnail name because it is always a jpg
	i.Thumbnail = fmt.Sprintf("%ss.jpg", filename)

	// set the full file path
	i.Filepath = filepath.Join(local.Settings.Directories.ImageDir, i.Filename)

	// set the full thumbnail path
	i.Thumbpath = filepath.Join(local.Settings.Directories.ThumbnailDir, i.Thumbnail)

	return

}

func (i *ImageType) createThumbnail(maxwidth, maxheight int) (err error) {

	var imagef string

	if i.video {
		imagef = fmt.Sprintf("%s[0]", i.Thumbpath)
	} else {
		imagef = fmt.Sprintf("%s[0]", i.Filepath)
	}

	originalDimensions := fmt.Sprintf("%dx%d", i.OrigWidth, i.OrigHeight)

	var args []string

	// different options for avatars
	if i.avatar {
		args = []string{
			"-size",
			originalDimensions,
			imagef,
			"-background",
			"none",
			"-thumbnail",
			fmt.Sprintf("%dx%d^", maxwidth, maxheight),
			"-gravity",
			"center",
			"-extent",
			fmt.Sprintf("%dx%d", maxwidth, maxheight),
			i.Thumbpath,
		}
	} else {
		args = []string{
			"-background",
			"white",
			"-flatten",
			"-size",
			originalDimensions,
			"-resize",
			fmt.Sprintf("%dx%d>", maxwidth, maxheight),
			"-quality",
			"90",
			imagef,
			i.Thumbpath,
		}
	}

	_, err = exec.Command("convert", args...).Output()
	if err != nil {
		return errors.New("Problem making thumbnail")
	}

	thumb, err := os.Open(i.Thumbpath)
	if err != nil {
		return errors.New("Problem creating thumbnail file")
	}
	defer thumb.Close()

	img, _, err := image.DecodeConfig(thumb)
	if err != nil {
		return errors.New("Problem decoding thumbnail")
	}

	i.ThumbWidth = img.Width
	i.ThumbHeight = img.Height

	return

}

func (i *ImageType) copyToS3() (err error) {

	s3 := amazon.New()

	err = s3.Save(i.Filepath, fmt.Sprintf("src/%s", i.Filename), i.mime, false)
	if err != nil {
		return
	}

	err = s3.Save(i.Thumbpath, fmt.Sprintf("thumb/%s", i.Thumbnail), "image/jpeg", false)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/addrs"
	"github.com/zclconf/go-cty/cty"
)

// NodePlannableResourceInstance represents a _single_ resource
// instance that is plannable. This means this represents a single
// count index, for example.
type NodePlannableResourceInstance struct {
	*NodeAbstractResourceInstance
}

var (
	_ GraphNodeSubPath              = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeReferenceable        = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeReferencer           = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeResource             = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeResourceInstance     = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeAttachResourceConfig = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeAttachResourceState  = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeEvalable             = (*NodePlannableResourceInstance)(nil)
)

// GraphNodeEvalable
func (n *NodePlannableResourceInstance) EvalTree() EvalNode {
	addr := n.ResourceInstanceAddr()

	// State still uses legacy-style internal ids, so we need to shim to get
	// a suitable key to use.
	stateId := NewLegacyResourceInstanceAddress(addr).stateId()

	// Determine the dependencies for the state.
	stateDeps := n.StateReferences()

	// Eval info is different depending on what kind of resource this is
	switch addr.Resource.Resource.Mode {
	case addrs.ManagedResourceMode:
		return n.evalTreeManagedResource(addr, stateId, stateDeps)
	case addrs.DataResourceMode:
		return n.evalTreeDataResource(addr, stateId, stateDeps)
	default:
		panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode))
	}
}

func (n *NodePlannableResourceInstance) evalTreeDataResource(addr addrs.AbsResourceInstance, stateId string, stateDeps []string) EvalNode {
	var provider ResourceProvider
	var providerSchema *ProviderSchema
	var diff *InstanceDiff
	var state *InstanceState
	var configVal cty.Value

	return &EvalSequence{
		Nodes: []EvalNode{
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},

			&EvalGetProvider{
				Addr:   n.ResolvedProvider,
				Output: &provider,
				Schema: &providerSchema,
			},

			&EvalReadDataDiff{
				Addr:           addr.Resource,
				Config:         n.Config,
				Provider:       &provider,
				ProviderSchema: &providerSchema,
				Output:         &diff,
				OutputValue:    &configVal,
				OutputState:    &state,
			},

			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					computed := !configVal.IsWhollyKnown()

					// If the configuration is complete and we
					// already have a state then we don't need to
					// do any further work during apply, because we
					// already populated the state during refresh.
					if !computed && state != nil {
						return true, EvalEarlyExitError{}
					}

					return true, nil
				},
				Then: EvalNoop{},
			},

			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.ResolvedProvider,
				Dependencies: stateDeps,
				State:        &state,
			},

			&EvalWriteDiff{
				Name: stateId,
				Diff: &diff,
			},
		},
	}
}

func (n *NodePlannableResourceInstance) evalTreeManagedResource(addr addrs.AbsResourceInstance, stateId string, stateDeps []string) EvalNode {
	var provider ResourceProvider
	var providerSchema *ProviderSchema
	var diff *InstanceDiff
	var state *InstanceState

	return &EvalSequence{
		Nodes: []EvalNode{
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},

			&EvalGetProvider{
				Addr:   n.ResolvedProvider,
				Output: &provider,
				Schema: &providerSchema,
			},

			&EvalDiff{
				Addr:           addr.Resource,
				Config:         n.Config,
				Provider:       &provider,
				ProviderSchema: &providerSchema,
				State:          &state,
				OutputDiff:     &diff,
				OutputState:    &state,
			},
			&EvalCheckPreventDestroy{
				Addr:   addr.Resource,
				Config: n.Config,
				Diff:   &diff,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.ResolvedProvider,
				Dependencies: stateDeps,
				State:        &state,
			},
			&EvalWriteDiff{
				Name: stateId,
				Diff: &diff,
			},
		},
	}
}
<commit_msg>core: fix NodePlannableResourceInstance data reads<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/addrs"
	"github.com/zclconf/go-cty/cty"
)

// NodePlannableResourceInstance represents a _single_ resource
// instance that is plannable. This means this represents a single
// count index, for example.
type NodePlannableResourceInstance struct {
	*NodeAbstractResourceInstance
}

var (
	_ GraphNodeSubPath              = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeReferenceable        = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeReferencer           = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeResource             = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeResourceInstance     = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeAttachResourceConfig = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeAttachResourceState  = (*NodePlannableResourceInstance)(nil)
	_ GraphNodeEvalable             = (*NodePlannableResourceInstance)(nil)
)

// GraphNodeEvalable
func (n *NodePlannableResourceInstance) EvalTree() EvalNode {
	addr := n.ResourceInstanceAddr()

	// State still uses legacy-style internal ids, so we need to shim to get
	// a suitable key to use.
	stateId := NewLegacyResourceInstanceAddress(addr).stateId()

	// Determine the dependencies for the state.
	stateDeps := n.StateReferences()

	// Eval info is different depending on what kind of resource this is
	switch addr.Resource.Resource.Mode {
	case addrs.ManagedResourceMode:
		return n.evalTreeManagedResource(addr, stateId, stateDeps)
	case addrs.DataResourceMode:
		return n.evalTreeDataResource(addr, stateId, stateDeps)
	default:
		panic(fmt.Errorf("unsupported resource mode %s", n.Config.Mode))
	}
}

func (n *NodePlannableResourceInstance) evalTreeDataResource(addr addrs.AbsResourceInstance, stateId string, stateDeps []string) EvalNode {
	var provider ResourceProvider
	var providerSchema *ProviderSchema
	var diff *InstanceDiff
	var state *InstanceState
	var configVal cty.Value

	return &EvalSequence{
		Nodes: []EvalNode{
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},

			&EvalGetProvider{
				Addr:   n.ResolvedProvider,
				Output: &provider,
				Schema: &providerSchema,
			},

			&EvalReadDataDiff{
				Addr:           addr.Resource,
				Config:         n.Config,
				Provider:       &provider,
				ProviderSchema: &providerSchema,
				Output:         &diff,
				OutputValue:    &configVal,
				OutputState:    &state,
			},

			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.ResolvedProvider,
				Dependencies: stateDeps,
				State:        &state,
			},

			&EvalWriteDiff{
				Name: stateId,
				Diff: &diff,
			},
		},
	}
}

func (n *NodePlannableResourceInstance) evalTreeManagedResource(addr addrs.AbsResourceInstance, stateId string, stateDeps []string) EvalNode {
	var provider ResourceProvider
	var providerSchema *ProviderSchema
	var diff *InstanceDiff
	var state *InstanceState

	return &EvalSequence{
		Nodes: []EvalNode{
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},

			&EvalGetProvider{
				Addr:   n.ResolvedProvider,
				Output: &provider,
				Schema: &providerSchema,
			},

			&EvalDiff{
				Addr:           addr.Resource,
				Config:         n.Config,
				Provider:       &provider,
				ProviderSchema: &providerSchema,
				State:          &state,
				OutputDiff:     &diff,
				OutputState:    &state,
			},
			&EvalCheckPreventDestroy{
				Addr:   addr.Resource,
				Config: n.Config,
				Diff:   &diff,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.ResolvedProvider,
				Dependencies: stateDeps,
				State:        &state,
			},
			&EvalWriteDiff{
				Name: stateId,
				Diff: &diff,
			},
		},
	}
}
<|endoftext|>


<commit_before>package utils

import "errors"

var (
	ErrNoID   = errors.New("No ID tag Found")
	ErrNoRevs = errors.New("No Rev tag Found")
)

const (
	Ignore    = "-"
	OmitEmpty = "omitempty"
)
<commit_msg>Move isStruct to utils package<commit_after>package utils

import (
	"errors"
	"fmt"
	"reflect"
)

var (
	ErrNoID       = errors.New("No ID tag Found")
	ErrNoRevs     = errors.New("No Rev tag Found")
	ErrNotAStruct = errors.New("Unsupported. Not a struct")
)

const (
	Ignore    = "-"
	OmitEmpty = "omitempty"
)

func IsStruct(typ reflect.Type) bool {

	if typ.Kind() != reflect.Struct {
		fmt.Println(typ.Kind())
		return false

	}
	return true
}
<|endoftext|>


<commit_before>package utils

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/user"
	"sort"
	"strconv"
	"time"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Book   string
	APIKey string
}

// Deprecated. See upgrade/migrate.go
type YAMLDnote map[string][]string

type Dnote map[string]Book
type Book []Note
type Note struct {
	UID     string
	Content string
	AddedOn int64
}

const configFilename = ".dnoterc"
const DnoteUpdateFilename = ".dnote-upgrade"
const dnoteFilename = ".dnote"
const Version = "0.1.0-beta.3"

const letterRunes = "abcdefghipqrstuvwxyz0123456789"

func init() {
	rand.Seed(time.Now().UnixNano())
}

func GenerateNoteID() string {
	result := make([]byte, 8)
	for i := range result {
		result[i] = letterRunes[rand.Intn(len(letterRunes))]
	}

	return string(result)
}

func GetConfigPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, configFilename), nil
}

func GetDnotePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, dnoteFilename), nil
}

func GetYAMLDnoteArchivePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, ".dnote-yaml-archived"), nil
}

func GenerateConfigFile() error {
	content := []byte("book: general\n")
	configPath, err := GetConfigPath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(configPath, content, 0644)
	return err
}

func TouchDnoteFile() error {
	dnotePath, err := GetDnotePath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(dnotePath, []byte{}, 0644)
	return err
}

func TouchDnoteUpgradeFile() error {
	dnoteUpdatePath, err := GetDnoteUpdatePath()
	if err != nil {
		return err
	}

	epoch := strconv.FormatInt(time.Now().Unix(), 10)
	content := []byte(fmt.Sprintf("LAST_UPGRADE_EPOCH: %s\n", epoch))

	err = ioutil.WriteFile(dnoteUpdatePath, content, 0644)
	return err
}

func GetDnoteUpdatePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, DnoteUpdateFilename), nil
}

func AskConfirmation(question string) (bool, error) {
	fmt.Printf("%s [Y/n]: ", question)

	reader := bufio.NewReader(os.Stdin)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}

	ok := res == "y\n" || res == "Y\n" || res == "\n"

	return ok, nil
}

// ReadNoteContent reads the content of dnote
func ReadNoteContent() ([]byte, error) {
	notePath, err := GetDnotePath()
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(notePath)
	if err != nil {
		return nil, err
	}

	return b, nil
}

// GetDnote reads and parses the dnote
func GetDnote() (Dnote, error) {
	ret := Dnote{}

	b, err := ReadNoteContent()
	if err != nil {
		return ret, err
	}

	err = json.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}

// WriteDnote persists the state of Dnote into the dnote file
func WriteDnote(dnote Dnote) error {
	d, err := json.MarshalIndent(dnote, "", "  ")
	if err != nil {
		return err
	}

	notePath, err := GetDnotePath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(notePath, d, 0644)
	if err != nil {
		return err
	}

	return nil
}

// Deprecated. See upgrade/upgrade.go
func GetNote() (YAMLDnote, error) {
	ret := YAMLDnote{}

	b, err := ReadNoteContent()
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}
func WriteConfig(config Config) error {
	d, err := yaml.Marshal(config)
	if err != nil {
		return err
	}

	configPath, err := GetConfigPath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(configPath, d, 0644)
	if err != nil {
		return err
	}

	return nil
}

func ReadConfig() (Config, error) {
	var ret Config

	configPath, err := GetConfigPath()
	if err != nil {
		return ret, err
	}

	b, err := ioutil.ReadFile(configPath)
	if err != nil {
		return ret, err
	}

	err = yaml.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}

func GetCurrentBook() (string, error) {
	config, err := ReadConfig()
	if err != nil {
		return "", err
	}

	return config.Book, nil
}

func GetBooks() ([]string, error) {
	dnote, err := GetDnote()
	if err != nil {
		return nil, err
	}

	books := make([]string, 0, len(dnote))
	for k := range dnote {
		books = append(books, k)
	}

	sort.Strings(books)

	return books, nil
}
<commit_msg>Bump version<commit_after>package utils

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/user"
	"sort"
	"strconv"
	"time"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Book   string
	APIKey string
}

// Deprecated. See upgrade/migrate.go
type YAMLDnote map[string][]string

type Dnote map[string]Book
type Book []Note
type Note struct {
	UID     string
	Content string
	AddedOn int64
}

const configFilename = ".dnoterc"
const DnoteUpdateFilename = ".dnote-upgrade"
const dnoteFilename = ".dnote"
const Version = "0.1.0"

const letterRunes = "abcdefghipqrstuvwxyz0123456789"

func init() {
	rand.Seed(time.Now().UnixNano())
}

func GenerateNoteID() string {
	result := make([]byte, 8)
	for i := range result {
		result[i] = letterRunes[rand.Intn(len(letterRunes))]
	}

	return string(result)
}

func GetConfigPath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, configFilename), nil
}

func GetDnotePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, dnoteFilename), nil
}

func GetYAMLDnoteArchivePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, ".dnote-yaml-archived"), nil
}

func GenerateConfigFile() error {
	content := []byte("book: general\n")
	configPath, err := GetConfigPath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(configPath, content, 0644)
	return err
}

func TouchDnoteFile() error {
	dnotePath, err := GetDnotePath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(dnotePath, []byte{}, 0644)
	return err
}

func TouchDnoteUpgradeFile() error {
	dnoteUpdatePath, err := GetDnoteUpdatePath()
	if err != nil {
		return err
	}

	epoch := strconv.FormatInt(time.Now().Unix(), 10)
	content := []byte(fmt.Sprintf("LAST_UPGRADE_EPOCH: %s\n", epoch))

	err = ioutil.WriteFile(dnoteUpdatePath, content, 0644)
	return err
}

func GetDnoteUpdatePath() (string, error) {
	usr, err := user.Current()
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s/%s", usr.HomeDir, DnoteUpdateFilename), nil
}

func AskConfirmation(question string) (bool, error) {
	fmt.Printf("%s [Y/n]: ", question)

	reader := bufio.NewReader(os.Stdin)
	res, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}

	ok := res == "y\n" || res == "Y\n" || res == "\n"

	return ok, nil
}

// ReadNoteContent reads the content of dnote
func ReadNoteContent() ([]byte, error) {
	notePath, err := GetDnotePath()
	if err != nil {
		return nil, err
	}

	b, err := ioutil.ReadFile(notePath)
	if err != nil {
		return nil, err
	}

	return b, nil
}

// GetDnote reads and parses the dnote
func GetDnote() (Dnote, error) {
	ret := Dnote{}

	b, err := ReadNoteContent()
	if err != nil {
		return ret, err
	}

	err = json.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}

// WriteDnote persists the state of Dnote into the dnote file
func WriteDnote(dnote Dnote) error {
	d, err := json.MarshalIndent(dnote, "", "  ")
	if err != nil {
		return err
	}

	notePath, err := GetDnotePath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(notePath, d, 0644)
	if err != nil {
		return err
	}

	return nil
}

// Deprecated. See upgrade/upgrade.go
func GetNote() (YAMLDnote, error) {
	ret := YAMLDnote{}

	b, err := ReadNoteContent()
	if err != nil {
		return nil, err
	}

	err = yaml.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}
func WriteConfig(config Config) error {
	d, err := yaml.Marshal(config)
	if err != nil {
		return err
	}

	configPath, err := GetConfigPath()
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(configPath, d, 0644)
	if err != nil {
		return err
	}

	return nil
}

func ReadConfig() (Config, error) {
	var ret Config

	configPath, err := GetConfigPath()
	if err != nil {
		return ret, err
	}

	b, err := ioutil.ReadFile(configPath)
	if err != nil {
		return ret, err
	}

	err = yaml.Unmarshal(b, &ret)
	if err != nil {
		return ret, err
	}

	return ret, nil
}

func GetCurrentBook() (string, error) {
	config, err := ReadConfig()
	if err != nil {
		return "", err
	}

	return config.Book, nil
}

func GetBooks() ([]string, error) {
	dnote, err := GetDnote()
	if err != nil {
		return nil, err
	}

	books := make([]string, 0, len(dnote))
	for k := range dnote {
		books = append(books, k)
	}

	sort.Strings(books)

	return books, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package certrefresh

import (
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/open-policy-agent/opa/test/e2e"
)

var testRuntime *e2e.TestRuntime
var pool *x509.CertPool

// print error to stderr, exit 1
func fatal(err interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", err)
	os.Exit(1)
}

const (
	certFile0    = "testdata/server-cert.pem"
	certKeyFile0 = "testdata/server-key.pem"
	serial0      = "481849676048721749484276160748693385016044597443"
	certFile1    = "testdata/server-cert-new.pem"
	certKeyFile1 = "testdata/server-key-new.pem"
	serial1      = "481849676048721749484276160748693385016044597444"
)

var certFile, certKeyFile string

func TestMain(m *testing.M) {
	flag.Parse()
	caCertPEM, err := ioutil.ReadFile("testdata/ca.pem")
	if err != nil {
		fatal(err)
	}
	pool = x509.NewCertPool()
	if ok := pool.AppendCertsFromPEM(caCertPEM); !ok {
		fatal("failed to parse CA cert")
	}

	tmp, err := os.MkdirTemp("", "e2e_certrefresh")
	if err != nil {
		fatal(err)
	}
	defer os.RemoveAll(tmp)

	certFile = filepath.Join(tmp, "server-cert.pem")
	if err := copy(certFile0, certFile); err != nil {
		fatal(err)
	}

	certKeyFile = filepath.Join(tmp, "server-key.pem")
	if err := copy(certKeyFile0, certKeyFile); err != nil {
		fatal(err)
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		fatal(err)
	}

	testServerParams := e2e.NewAPIServerTestParams()
	testServerParams.Addrs = &[]string{"https://127.0.0.1:0"}
	testServerParams.CertPool = pool
	testServerParams.Certificate = &cert
	testServerParams.CertificateFile = certFile
	testServerParams.CertificateKeyFile = certKeyFile
	testServerParams.CertificateRefresh = time.Millisecond

	testRuntime, err = e2e.NewTestRuntime(testServerParams)
	if err != nil {
		fatal(err)
	}

	// We need a client with proper TLS setup, otherwise the health check
	// that loops to determine if the server is ready will fail.
	testRuntime.Client = newClient()

	os.Exit(testRuntime.RunTests(m))
}

func TestCertificateRotation(t *testing.T) {
	wait := 20 * time.Millisecond // file reload happens every millisecond

	// before rotation
	cert := getCert(t)
	if exp, act := serial0, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// replace file on disk
	replaceCerts(t, certFile1, certKeyFile1)
	time.Sleep(wait)

	// after rotation
	cert = getCert(t)
	if exp, act := serial1, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// replace file with nothing
	replaceCerts(t, os.DevNull, os.DevNull)
	time.Sleep(wait)

	// second cert still used
	cert = getCert(t)
	if exp, act := serial1, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// go back to first cert
	replaceCerts(t, certFile0, certKeyFile0)
	time.Sleep(wait)
	cert = getCert(t)
	if exp, act := serial0, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}
}

func newClient() *http.Client {
	c := *http.DefaultClient
	tr := http.DefaultTransport.(*http.Transport).Clone()
	tr.TLSClientConfig = &tls.Config{
		RootCAs: pool,
	}
	c.Transport = tr
	return &c
}

func copy(from, to string) error {
	src, err := os.Open(from)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.Create(to)
	if err != nil {
		return err
	}
	defer dst.Close()

	_, err = io.Copy(dst, src)
	return err
}

func getCert(t *testing.T) *x509.Certificate {
	t.Helper()
	u, err := url.Parse(testRuntime.URL())
	if err != nil {
		t.Fatal(err)
	}
	c := newClient()
	cfg := c.Transport.(*http.Transport).TLSClientConfig
	conn, err := tls.Dial("tcp", u.Host, cfg)
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	defer conn.Close()

	return conn.ConnectionState().PeerCertificates[0]
}

func replaceCerts(t *testing.T, cert, key string) {
	t.Helper()

	if err := copy(cert, certFile); err != nil {
		t.Fatal(err)
	}
	if err := copy(key, certKeyFile); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>test/e2e/certrefresh: Add file.Sync() to eliminate slow write failures. (#5033)<commit_after>// Copyright 2021 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package certrefresh

import (
	"crypto/tls"
	"crypto/x509"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/open-policy-agent/opa/test/e2e"
)

var testRuntime *e2e.TestRuntime
var pool *x509.CertPool

// print error to stderr, exit 1
func fatal(err interface{}) {
	fmt.Fprintf(os.Stderr, "%s\n", err)
	os.Exit(1)
}

const (
	certFile0    = "testdata/server-cert.pem"
	certKeyFile0 = "testdata/server-key.pem"
	serial0      = "481849676048721749484276160748693385016044597443"
	certFile1    = "testdata/server-cert-new.pem"
	certKeyFile1 = "testdata/server-key-new.pem"
	serial1      = "481849676048721749484276160748693385016044597444"
)

var certFile, certKeyFile string

func TestMain(m *testing.M) {
	flag.Parse()
	caCertPEM, err := ioutil.ReadFile("testdata/ca.pem")
	if err != nil {
		fatal(err)
	}
	pool = x509.NewCertPool()
	if ok := pool.AppendCertsFromPEM(caCertPEM); !ok {
		fatal("failed to parse CA cert")
	}

	tmp, err := os.MkdirTemp("", "e2e_certrefresh")
	if err != nil {
		fatal(err)
	}
	defer os.RemoveAll(tmp)

	certFile = filepath.Join(tmp, "server-cert.pem")
	if err := copy(certFile0, certFile); err != nil {
		fatal(err)
	}

	certKeyFile = filepath.Join(tmp, "server-key.pem")
	if err := copy(certKeyFile0, certKeyFile); err != nil {
		fatal(err)
	}

	cert, err := tls.LoadX509KeyPair(certFile, certKeyFile)
	if err != nil {
		fatal(err)
	}

	testServerParams := e2e.NewAPIServerTestParams()
	testServerParams.Addrs = &[]string{"https://127.0.0.1:0"}
	testServerParams.CertPool = pool
	testServerParams.Certificate = &cert
	testServerParams.CertificateFile = certFile
	testServerParams.CertificateKeyFile = certKeyFile
	testServerParams.CertificateRefresh = time.Millisecond

	testRuntime, err = e2e.NewTestRuntime(testServerParams)
	if err != nil {
		fatal(err)
	}

	// We need a client with proper TLS setup, otherwise the health check
	// that loops to determine if the server is ready will fail.
	testRuntime.Client = newClient()

	os.Exit(testRuntime.RunTests(m))
}

func TestCertificateRotation(t *testing.T) {
	wait := 20 * time.Millisecond // file reload happens every millisecond

	// before rotation
	cert := getCert(t)
	if exp, act := serial0, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// replace file on disk
	replaceCerts(t, certFile1, certKeyFile1)
	time.Sleep(wait)

	// after rotation
	cert = getCert(t)
	if exp, act := serial1, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// replace file with nothing
	replaceCerts(t, os.DevNull, os.DevNull)
	time.Sleep(wait)

	// second cert still used
	cert = getCert(t)
	if exp, act := serial1, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}

	// go back to first cert
	replaceCerts(t, certFile0, certKeyFile0)
	time.Sleep(wait)
	cert = getCert(t)
	if exp, act := serial0, string(cert.SerialNumber.String()); exp != act {
		t.Fatalf("expected signature %s, got %s", exp, act)
	}
}

func newClient() *http.Client {
	c := *http.DefaultClient
	tr := http.DefaultTransport.(*http.Transport).Clone()
	tr.TLSClientConfig = &tls.Config{
		RootCAs: pool,
	}
	c.Transport = tr
	return &c
}

func copy(from, to string) error {
	src, err := os.Open(from)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.Create(to)
	if err != nil {
		return err
	}
	defer dst.Close()

	_, err = io.Copy(dst, src)
	if err != nil {
		return err
	}

	// Ensure that our writes get committed to disk, even on slower systems.
	return dst.Sync()
}

func getCert(t *testing.T) *x509.Certificate {
	t.Helper()
	u, err := url.Parse(testRuntime.URL())
	if err != nil {
		t.Fatal(err)
	}
	c := newClient()
	cfg := c.Transport.(*http.Transport).TLSClientConfig
	conn, err := tls.Dial("tcp", u.Host, cfg)
	if err != nil {
		t.Fatalf("dial: %v", err)
	}
	defer conn.Close()

	return conn.ConnectionState().PeerCertificates[0]
}

func replaceCerts(t *testing.T, cert, key string) {
	t.Helper()

	if err := copy(cert, certFile); err != nil {
		t.Fatal(err)
	}
	if err := copy(key, certKeyFile); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package system

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/errors"
)

var _ Validator = &KernelValidator{}

// KernelValidator validates kernel. Currently only validate kernel version
// and kernel configuration.
type KernelValidator struct {
	kernelRelease string
	Reporter      Reporter
}

func (k *KernelValidator) Name() string {
	return "kernel"
}

// kConfigOption is the possible kernel config option.
type kConfigOption string

const (
	builtIn  kConfigOption = "y"
	asModule kConfigOption = "m"
	leftOut  kConfigOption = "n"

	// validKConfigRegex is the regex matching kernel configuration line.
	validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]"
	// kConfigPrefix is the prefix of kernel configuration.
	kConfigPrefix = "CONFIG_"
)

func (k *KernelValidator) Validate(spec SysSpec) (error, error) {
	release, err := exec.Command("uname", "-r").CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("failed to get kernel release: %v", err)
	}
	k.kernelRelease = strings.TrimSpace(string(release))
	var errs []error
	errs = append(errs, k.validateKernelVersion(spec.KernelSpec))
	errs = append(errs, k.validateKernelConfig(spec.KernelSpec))
	return nil, errors.NewAggregate(errs)
}

// validateKernelVersion validates the kernel version.
func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error {
	glog.Infof("Validating kernel version")
	versionRegexps := kSpec.Versions
	for _, versionRegexp := range versionRegexps {
		r := regexp.MustCompile(versionRegexp)
		if r.MatchString(k.kernelRelease) {
			k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, good)
			return nil
		}
	}
	k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, bad)
	return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease)
}

// validateKernelConfig validates the kernel configurations.
func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error {
	glog.Infof("Validating kernel config")
	allConfig, err := k.getKernelConfig()
	if err != nil {
		return fmt.Errorf("failed to parse kernel config: %v", err)
	}
	return k.validateCachedKernelConfig(allConfig, kSpec)
}

// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type.
func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error {
	badConfigs := []string{}
	// reportAndRecord is a helper function to record bad config when
	// report.
	reportAndRecord := func(name, msg, desc string, result ValidationResultType) {
		if result == bad {
			badConfigs = append(badConfigs, name)
		}
		// report description when the config is bad or warn.
		if result != good && desc != "" {
			msg = msg + " - " + desc
		}
		k.Reporter.Report(name, msg, result)
	}
	const (
		required = iota
		optional
		forbidden
	)
	validateOpt := func(config KernelConfig, expect int) {
		var found, missing ValidationResultType
		switch expect {
		case required:
			found, missing = good, bad
		case optional:
			found, missing = good, warn
		case forbidden:
			found, missing = bad, good
		}
		var name string
		var opt kConfigOption
		var ok bool
		for _, name = range append([]string{config.Name}, config.Aliases...) {
			name = kConfigPrefix + name
			if opt, ok = allConfig[name]; ok {
				break
			}
		}
		if !ok {
			reportAndRecord(name, "not set", config.Description, missing)
			return
		}
		switch opt {
		case builtIn:
			reportAndRecord(name, "enabled", config.Description, found)
		case asModule:
			reportAndRecord(name, "enabled (as module)", config.Description, found)
		case leftOut:
			reportAndRecord(name, "disabled", config.Description, missing)
		default:
			reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing)
		}
	}
	for _, config := range kSpec.Required {
		validateOpt(config, required)
	}
	for _, config := range kSpec.Optional {
		validateOpt(config, optional)
	}
	for _, config := range kSpec.Forbidden {
		validateOpt(config, forbidden)
	}
	if len(badConfigs) > 0 {
		return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " "))
	}
	return nil
}

// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config
// file is found it will read the configurations into a byte buffer and return. If the kernel
// config file is not found, it will try to load kernel config module and retry again.
func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) {
	possibePaths := []string{
		"/proc/config.gz",
		"/boot/config-" + k.kernelRelease,
		"/usr/src/linux-" + k.kernelRelease + "/.config",
		"/usr/src/linux/.config",
	}
	configsModule := "configs"
	modprobeCmd := "modprobe"
	// loadModule indicates whether we've tried to load kernel config module ourselves.
	loadModule := false
	for {
		for _, path := range possibePaths {
			_, err := os.Stat(path)
			if err != nil {
				continue
			}
			// Buffer the whole file, so that we can close the file and unload
			// kernel config module in this function.
			b, err := ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
			var r io.Reader
			r = bytes.NewReader(b)
			// This is a gzip file (config.gz), unzip it.
			if filepath.Ext(path) == ".gz" {
				r, err = gzip.NewReader(r)
				if err != nil {
					return nil, err
				}
			}
			return r, nil
		}
		// If we've tried to load kernel config module, break and return error.
		if loadModule {
			break
		}
		// If the kernel config file is not found, try to load the kernel
		// config module and check again.
		output, err := exec.Command(modprobeCmd, configsModule).CombinedOutput()
		if err != nil {
			return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v",
				configsModule, output, err)
		}
		// Unload the kernel config module to make sure the validation have no side effect.
		defer exec.Command(modprobeCmd, "-r", configsModule).Run()
		loadModule = true
	}
	return nil, fmt.Errorf("no config path in %v is available", possibePaths)
}

// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type.
func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) {
	r, err := k.getKernelConfigReader()
	if err != nil {
		return nil, err
	}
	return k.parseKernelConfig(r)
}

// parseKernelConfig converts kernel config to internal type.
func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) {
	config := map[string]kConfigOption{}
	regex := regexp.MustCompile(validKConfigRegex)
	s := bufio.NewScanner(r)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return nil, err
		}
		line := strings.TrimSpace(s.Text())
		if !regex.MatchString(line) {
			continue
		}
		fields := strings.Split(line, "=")
		if len(fields) != 2 {
			glog.Errorf("Unexpected fields number in config %q", line)
			continue
		}
		config[fields[0]] = kConfigOption(fields[1])
	}
	return config, nil

}
<commit_msg>add kernel config locations for fedora and atomic<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package system

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/errors"
)

var _ Validator = &KernelValidator{}

// KernelValidator validates kernel. Currently only validate kernel version
// and kernel configuration.
type KernelValidator struct {
	kernelRelease string
	Reporter      Reporter
}

func (k *KernelValidator) Name() string {
	return "kernel"
}

// kConfigOption is the possible kernel config option.
type kConfigOption string

const (
	builtIn  kConfigOption = "y"
	asModule kConfigOption = "m"
	leftOut  kConfigOption = "n"

	// validKConfigRegex is the regex matching kernel configuration line.
	validKConfigRegex = "^CONFIG_[A-Z0-9_]+=[myn]"
	// kConfigPrefix is the prefix of kernel configuration.
	kConfigPrefix = "CONFIG_"
)

func (k *KernelValidator) Validate(spec SysSpec) (error, error) {
	release, err := exec.Command("uname", "-r").CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("failed to get kernel release: %v", err)
	}
	k.kernelRelease = strings.TrimSpace(string(release))
	var errs []error
	errs = append(errs, k.validateKernelVersion(spec.KernelSpec))
	errs = append(errs, k.validateKernelConfig(spec.KernelSpec))
	return nil, errors.NewAggregate(errs)
}

// validateKernelVersion validates the kernel version.
func (k *KernelValidator) validateKernelVersion(kSpec KernelSpec) error {
	glog.Infof("Validating kernel version")
	versionRegexps := kSpec.Versions
	for _, versionRegexp := range versionRegexps {
		r := regexp.MustCompile(versionRegexp)
		if r.MatchString(k.kernelRelease) {
			k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, good)
			return nil
		}
	}
	k.Reporter.Report("KERNEL_VERSION", k.kernelRelease, bad)
	return fmt.Errorf("unsupported kernel release: %s", k.kernelRelease)
}

// validateKernelConfig validates the kernel configurations.
func (k *KernelValidator) validateKernelConfig(kSpec KernelSpec) error {
	glog.Infof("Validating kernel config")
	allConfig, err := k.getKernelConfig()
	if err != nil {
		return fmt.Errorf("failed to parse kernel config: %v", err)
	}
	return k.validateCachedKernelConfig(allConfig, kSpec)
}

// validateCachedKernelConfig validates the kernel confgiurations cached in internal data type.
func (k *KernelValidator) validateCachedKernelConfig(allConfig map[string]kConfigOption, kSpec KernelSpec) error {
	badConfigs := []string{}
	// reportAndRecord is a helper function to record bad config when
	// report.
	reportAndRecord := func(name, msg, desc string, result ValidationResultType) {
		if result == bad {
			badConfigs = append(badConfigs, name)
		}
		// report description when the config is bad or warn.
		if result != good && desc != "" {
			msg = msg + " - " + desc
		}
		k.Reporter.Report(name, msg, result)
	}
	const (
		required = iota
		optional
		forbidden
	)
	validateOpt := func(config KernelConfig, expect int) {
		var found, missing ValidationResultType
		switch expect {
		case required:
			found, missing = good, bad
		case optional:
			found, missing = good, warn
		case forbidden:
			found, missing = bad, good
		}
		var name string
		var opt kConfigOption
		var ok bool
		for _, name = range append([]string{config.Name}, config.Aliases...) {
			name = kConfigPrefix + name
			if opt, ok = allConfig[name]; ok {
				break
			}
		}
		if !ok {
			reportAndRecord(name, "not set", config.Description, missing)
			return
		}
		switch opt {
		case builtIn:
			reportAndRecord(name, "enabled", config.Description, found)
		case asModule:
			reportAndRecord(name, "enabled (as module)", config.Description, found)
		case leftOut:
			reportAndRecord(name, "disabled", config.Description, missing)
		default:
			reportAndRecord(name, fmt.Sprintf("unknown option: %s", opt), config.Description, missing)
		}
	}
	for _, config := range kSpec.Required {
		validateOpt(config, required)
	}
	for _, config := range kSpec.Optional {
		validateOpt(config, optional)
	}
	for _, config := range kSpec.Forbidden {
		validateOpt(config, forbidden)
	}
	if len(badConfigs) > 0 {
		return fmt.Errorf("unexpected kernel config: %s", strings.Join(badConfigs, " "))
	}
	return nil
}

// getKernelConfigReader search kernel config file in a predefined list. Once the kernel config
// file is found it will read the configurations into a byte buffer and return. If the kernel
// config file is not found, it will try to load kernel config module and retry again.
func (k *KernelValidator) getKernelConfigReader() (io.Reader, error) {
	possibePaths := []string{
		"/proc/config.gz",
		"/boot/config-" + k.kernelRelease,
		"/usr/src/linux-" + k.kernelRelease + "/.config",
		"/usr/src/linux/.config",
		"/usr/lib/modules/" + k.kernelRelease + "/config",
		"/usr/lib/ostree-boot/config-" + k.kernelRelease,
	}
	configsModule := "configs"
	modprobeCmd := "modprobe"
	// loadModule indicates whether we've tried to load kernel config module ourselves.
	loadModule := false
	for {
		for _, path := range possibePaths {
			_, err := os.Stat(path)
			if err != nil {
				continue
			}
			// Buffer the whole file, so that we can close the file and unload
			// kernel config module in this function.
			b, err := ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
			var r io.Reader
			r = bytes.NewReader(b)
			// This is a gzip file (config.gz), unzip it.
			if filepath.Ext(path) == ".gz" {
				r, err = gzip.NewReader(r)
				if err != nil {
					return nil, err
				}
			}
			return r, nil
		}
		// If we've tried to load kernel config module, break and return error.
		if loadModule {
			break
		}
		// If the kernel config file is not found, try to load the kernel
		// config module and check again.
		output, err := exec.Command(modprobeCmd, configsModule).CombinedOutput()
		if err != nil {
			return nil, fmt.Errorf("unable to load kernel module %q: output - %q, err - %v",
				configsModule, output, err)
		}
		// Unload the kernel config module to make sure the validation have no side effect.
		defer exec.Command(modprobeCmd, "-r", configsModule).Run()
		loadModule = true
	}
	return nil, fmt.Errorf("no config path in %v is available", possibePaths)
}

// getKernelConfig gets kernel config from kernel config file and convert kernel config to internal type.
func (k *KernelValidator) getKernelConfig() (map[string]kConfigOption, error) {
	r, err := k.getKernelConfigReader()
	if err != nil {
		return nil, err
	}
	return k.parseKernelConfig(r)
}

// parseKernelConfig converts kernel config to internal type.
func (k *KernelValidator) parseKernelConfig(r io.Reader) (map[string]kConfigOption, error) {
	config := map[string]kConfigOption{}
	regex := regexp.MustCompile(validKConfigRegex)
	s := bufio.NewScanner(r)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return nil, err
		}
		line := strings.TrimSpace(s.Text())
		if !regex.MatchString(line) {
			continue
		}
		fields := strings.Split(line, "=")
		if len(fields) != 2 {
			glog.Errorf("Unexpected fields number in config %q", line)
			continue
		}
		config[fields[0]] = kConfigOption(fields[1])
	}
	return config, nil

}
<|endoftext|>


<commit_before>// Package colonycore define the struct that needed by other package in colony
package colonycore

import (
	/*"github.com/eaciit/errorlib"
		// "github.com/eaciit/orm/v1"
		// "log"
		"mime/multipart"
		"strconv"
		"strings"
	    "github.com/eaciit/toolkit"
	*/
	"time"

	"github.com/eaciit/orm/v1"
	tk "github.com/eaciit/toolkit"
)

const (
/*ACTION_TYPE_HIVE     = "HIVE"
ACTION_TYPE_HDFS     = "HDFS"
ACTION_TYPE_SPARK    = "SPARK"
ACTION_TYPE_DECISION = "DECISION"
ACTION_TYPE_SSH      = "SSH"
ACTION_TYPE_KAFKA    = "KAFKA"

FORK_TYPE_ALL       = "ALL"
FORK_TYPE_ONE       = "ONE"
FORK_TYPE_MANDATORY = "MANDATORY"

SSH_OPERATION_MKDIR = "MKDIR"*/
// SSH_OPERATION_* please define
)

// DataFlow to define the flow name and description
// and also have the list of the actions inside the flow
// Actions list of action, the content of the action can be FlowAction or "list of FlowAction" -> for fork action
type DataFlow struct {
	orm.ModelBase
	ID           string       `json:"_id",bson:"_id"`
	Name         string       `json:"name"`
	Description  string       `json:"description"`
	CreatedDate  time.Time    `json:"createddate"`
	LastModified time.Time    `json:"lastmodified"`
	CreatedBy    string       `json:"createdby"`
	Actions      []FlowAction `json:"actions"`
	DataShapes   tk.M         `json:"datashapes"`
	GlobalParam  tk.M
}

func (c *DataFlow) TableName() string {
	return "dataflow"
}

func (c *DataFlow) RecordID() interface{} {
	return c.ID
}

/*// ActionBridge, to define the input and output of the previous action and next action
// PrevAction, ID of the previouse action
// NextAction, ID of the next action
// MappingParam, define the output and input paremeter name of the previous and next action
//      e.g.:
//              OUTPUT                INPUT
//              "id"                : "id"
//              "name"              : "name"
//              "address"           : "note"
//              "tag"               : "global.tag"
//              "global.address"    : "address"
type Bridge struct {
	Id           string
	Type         string
	PrevAction   string
	NextAction   string
	MappingParam tk.M
}*/

// FlowAction define the action that exist
// Type refer to cons ACTION_TYPE_*
// Action define the type of action e.g. Spark, HDFS, etc
// Server define which server for the action
// OK next step to go if the result is OK
// KO next step to go if the result is KO or NOT OK or ERROR
// Retry how many time the system will do the process if ERROR happen and then go to KO condition, default will be 3
// Wait interval in minute, default will be 1
// InputParam, will be the list of input that needed by the action
// OutputParam, will be the list of output from the action
// OutputType, will be the type of the output (text, json, *sv, xml)
// OutputPath, path of the output in the hdfs if empty then consider as stdout
type FlowAction struct {
	Id          string      `json:"_id"`
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Type        string      `json:"type"`
	Server      Server      `json:"server"`
	Action      interface{} `json:"action"`
	OK          []string    `json:"ok"`
	KO          []string    `json:"ko"`
	Retry       int         `json:"retry"`
	Interval    int         `json:"interval"`
	FirstAction bool
	InputParam  tk.M
	OutputParam tk.M
	OutputType  string
	OutputPath  string
}

// ActionHive action for HIVE
// ScriptPath path of the hive script to run
type ActionHive struct {
	ScriptPath string
	Params     []string
}

// ActionHDFS action for HDFS type for server
// Operation type operation
// The HDFS connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionHDFS struct {
	/*Operation  string `json:"operation"`
	Path       string `json:"path"`
	NewPath    string `json:"newpath"`
	Permission string
	User       string
	Group      string*/
	Command string
}

// ActionSSH action for SSH type of server
// Operation type operation
// The ssh connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionSSH struct {
	Command string
}

// ActionSpark action for SPARK
// Type of the code upload e.g. Java, Scala
// Mode e.g. client, cluster
// Args for other arguments, e.g. --executor-memory 2G
// Application the spark code will be mantain as application
type ActionSpark struct {
	Type   string `json: "type"`
	Master string
	Mode   string
	File   string
	// AppName   string
	MainClass string
	// Args      []string
	Args string
}

// ActionSpark action for Application
// Type e.g. Java, Scala, GO
// Application refer to application page
// when creating the action with type = application then should follow the application screen standard
type ActionApplication struct {
	Type        string `json: "type"`
	Application Application
}

// ActionHadoopStreaming for hadoop streaming
// Mapper name
// Reducer name
// Input can be folder or file
// Output folder
// Files list of file inside the hdfs
// the server type should be hdfs
type ActionHadoopStreaming struct {
	Jar     string
	Mapper  string
	Reducer string
	Input   string
	Output  string
	Files   []string
	// Params  []string
	Params string
}

// ActionKafka action for KAFKA
type ActionKafka struct {
	// other field will be different for each action type
}

/*
Doesn't need the Queestion, can be achived by using OK and KO or using decision
type ActionQuestion struct {
	Question string `json:"question"`
	Yes      string `json:"yes"`
	No       string `json:"no"`
	// other field will be different for each action type
}*/

// ActionDecision action to define the list of decision that action have
type ActionDecision struct {
	Conditions []Condition `json:"conditions"`
	// other field will be different for each action type
}

// Condition condition for the action decision
type Condition struct {
	Result     string `json:"result"`
	FlowAction string `json:"flowaction"`
}

type ActionStop struct {
	Message string `json:"message"`
}

type ActionShellScript struct {
	Script string `json:"script"`
}

type ActionJavaApp struct {
	Jar string `json:"jar"`
}

type ActionEmail struct {
	To      string `json:"to"`
	Subject string `json:"subject"`
	Body    string `json:"body"`
}

// ActionFork to define the forking condition
// Type refer to cons FORK_TYPE_*
type ActionFork struct {
	Actions []FlowAction `json:"actions"`
	Type    string       `json: "type"`
}

type DataFlowProcess struct {
	orm.ModelBase
	Id          string `json:"_id"`
	Flow        DataFlow
	Steps       []FlowAction
	StartDate   time.Time
	EndDate     time.Time
	StartedBy   string
	GlobalParam tk.M
}

func (c *DataFlowProcess) TableName() string {
	return "dataflowprocess"
}

func (c *DataFlowProcess) RecordID() interface{} {
	return c.Id
}
<commit_msg>change some struct fields<commit_after>// Package colonycore define the struct that needed by other package in colony
package colonycore

import (
	/*"github.com/eaciit/errorlib"
		// "github.com/eaciit/orm/v1"
		// "log"
		"mime/multipart"
		"strconv"
		"strings"
	    "github.com/eaciit/toolkit"
	*/
	"time"

	"github.com/eaciit/orm/v1"
	tk "github.com/eaciit/toolkit"
)

const (
/*ACTION_TYPE_HIVE     = "HIVE"
ACTION_TYPE_HDFS     = "HDFS"
ACTION_TYPE_SPARK    = "SPARK"
ACTION_TYPE_DECISION = "DECISION"
ACTION_TYPE_SSH      = "SSH"
ACTION_TYPE_KAFKA    = "KAFKA"

FORK_TYPE_ALL       = "ALL"
FORK_TYPE_ONE       = "ONE"
FORK_TYPE_MANDATORY = "MANDATORY"

SSH_OPERATION_MKDIR = "MKDIR"*/
// SSH_OPERATION_* please define
)

// DataFlow to define the flow name and description
// and also have the list of the actions inside the flow
// Actions list of action, the content of the action can be FlowAction or "list of FlowAction" -> for fork action
type DataFlow struct {
	orm.ModelBase
	ID           string       `json:"_id",bson:"_id"`
	Name         string       `json:"name"`
	Description  string       `json:"description"`
	CreatedDate  time.Time    `json:"createddate"`
	LastModified time.Time    `json:"lastmodified"`
	CreatedBy    string       `json:"createdby"`
	Actions      []FlowAction `json:"actions"`
	DataShapes   tk.M         `json:"datashapes"`
	GlobalParam  tk.M
}

func (c *DataFlow) TableName() string {
	return "dataflow"
}

func (c *DataFlow) RecordID() interface{} {
	return c.ID
}

/*// ActionBridge, to define the input and output of the previous action and next action
// PrevAction, ID of the previouse action
// NextAction, ID of the next action
// MappingParam, define the output and input paremeter name of the previous and next action
//      e.g.:
//              OUTPUT                INPUT
//              "id"                : "id"
//              "name"              : "name"
//              "address"           : "note"
//              "tag"               : "global.tag"
//              "global.address"    : "address"
type Bridge struct {
	Id           string
	Type         string
	PrevAction   string
	NextAction   string
	MappingParam tk.M
}*/

// FlowAction define the action that exist
// Type refer to cons ACTION_TYPE_*
// Action define the type of action e.g. Spark, HDFS, etc
// Server define which server for the action
// OK next step to go if the result is OK
// KO next step to go if the result is KO or NOT OK or ERROR
// Retry how many time the system will do the process if ERROR happen and then go to KO condition, default will be 3
// Wait interval in minute, default will be 1
// InputParam, will be the list of input that needed by the action
// OutputParam, will be the list of output from the action
// OutputType, will be the type of the output (text, json, *sv, xml)
// OutputPath, path of the output in the hdfs if empty then consider as stdout
type FlowAction struct {
	Id          string      `json:"_id"`
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Type        string      `json:"type"`
	Server      Server      `json:"server"`
	Action      interface{} `json:"action"`
	OK          []string    `json:"ok"`
	KO          []string    `json:"ko"`
	Retry       int         `json:"retry"`
	Interval    int         `json:"interval"`
	FirstAction bool
	InputParam  tk.M
	OutputParam tk.M
	OutputType  string
	OutputPath  string
}

// ActionHive action for HIVE
// ScriptPath path of the hive script to run
type ActionHive struct {
	ScriptPath string
	Params     []string
}

// ActionHDFS action for HDFS type for server
// Operation type operation
// The HDFS connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionHDFS struct {
	/*Operation  string `json:"operation"`
	Path       string `json:"path"`
	NewPath    string `json:"newpath"`
	Permission string
	User       string
	Group      string*/
	Command string
}

// ActionSSH action for SSH type of server
// Operation type operation
// The ssh connection will be provided by the server from FlowAction
// Path can be used for the current path, or old path, also can included the filename
// NewPath can be used for move, copy, etc
// Permission e.g. rwx-r--r-
type ActionSSH struct {
	Command string
}

// ActionSpark action for SPARK
// Type of the code upload e.g. Java, Scala
// Mode e.g. client, cluster
// Args for other arguments, e.g. --executor-memory 2G
// Application the spark code will be mantain as application
type ActionSpark struct {
	Type   string `json: "type"`
	Master string
	Mode   string
	File   string
	// AppName   string
	MainClass string
	// Args      []string
	Args string
}

// ActionSpark action for Application
// Type e.g. Java, Scala, GO
// Application refer to application page
// when creating the action with type = application then should follow the application screen standard
type ActionApplication struct {
	Type        string `json: "type"`
	Application Application
}

// ActionHadoopStreaming for hadoop streaming
// Mapper name
// Reducer name
// Input can be folder or file
// Output folder
// Files list of file inside the hdfs
// the server type should be hdfs
type ActionHadoopStreaming struct {
	Jar     string
	Mapper  string
	Reducer string
	Input   string
	Output  string
	Files   []string
	// Params  []string
	Params string
}

// ActionKafka action for KAFKA
type ActionKafka struct {
	// other field will be different for each action type
}

/*
Doesn't need the Queestion, can be achived by using OK and KO or using decision
type ActionQuestion struct {
	Question string `json:"question"`
	Yes      string `json:"yes"`
	No       string `json:"no"`
	// other field will be different for each action type
}*/

// ActionDecision action to define the list of decision that action have
type ActionDecision struct {
	Conditions []Condition `json:"conditions"`
	// other field will be different for each action type
}

// Condition condition for the action decision
type Condition struct {
	Result     string `json:"result"`
	FlowAction string `json:"flowaction"`
}

type ActionStop struct {
	Message string `json:"message"`
}

type ActionShellScript struct {
	Script string `json:"script"`
}

type ActionJavaApp struct {
	Jar string `json:"jar"`
}

type ActionEmail struct {
	To      string `json:"to"`
	CC      string `json:"to"`
	Subject string `json:"subject"`
	Body    string `json:"body"`
}

// ActionFork to define the forking condition
// Type refer to cons FORK_TYPE_*
type ActionFork struct {
	Actions []FlowAction `json:"actions"`
	Type    string       `json: "type"`
}

type DataFlowProcess struct {
	orm.ModelBase
	Id          string `json:"_id"`
	Flow        DataFlow
	Steps       []FlowAction
	StartDate   time.Time
	EndDate     time.Time
	StartedBy   string
	GlobalParam tk.M
}

func (c *DataFlowProcess) TableName() string {
	return "dataflowprocess"
}

func (c *DataFlowProcess) RecordID() interface{} {
	return c.Id
}
<|endoftext|>