<commit_before>package main

import (
	getopt "code.google.com/p/getopt"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	http "net/http"
	url "net/url"
	"os"
	"strconv"
	"time"
)

const FPASTE_URL = "http://fpaste.org"

func main() {
	opts := initConfig(os.Args)
	if *opts.help {
		getopt.PrintUsage(os.Stdout)
	} else {
		/*Passing stdin for test mocking*/
		files, errs := handleArgs(os.Stdin, getopt.CommandLine)
		for _, file := range files {
			if len(file) != 0 {
				if err := copyPaste(file, opts); err != nil {
					errs = append(errs, err)
				}
			}
		}
		for _, err := range errs {
			log.Print(err)
		}
		if len(errs) > 0 {
			os.Exit(-1)
		}
	}
}

type config struct {
	help   *bool
	priv   *bool
	user   *string
	pass   *string
	lang   *string
	expire *string
}

func initConfig(args []string) *config {
	getopt.CommandLine = getopt.New()
	var flags config
	flags.help = getopt.BoolLong("help", 'h', "Display this help")
	flags.priv = getopt.BoolLong("private", 'P', "Private paste flag")
	flags.user = getopt.StringLong("user", 'u', "", "An alphanumeric username of the paste author")
	flags.pass = getopt.StringLong("pass", 'p', "", "Add a password")
	flags.lang = getopt.StringLong("lang", 'l', "Text", "The development language used")
	flags.expire = getopt.StringLong("expire", 'e', "", "Seconds after which paste will be deleted from server")
	getopt.SetParameters("[FILE...]")
	getopt.CommandLine.Parse(args)
	return &flags
}

func handleArgs(stdin io.Reader, commandLine *getopt.Set) (files [][]byte, errs []error) {
	if commandLine.NArgs() > 0 {
		for _, x := range commandLine.Args() {
			file, err := os.Open(x)
			if err != nil {
				errs = append(errs, fmt.Errorf("Skipping [FILE: %s] since it cannot be opened (%s)", x, err))
				continue
			}
			defer file.Close()
			data, erread := ioutil.ReadAll(file)
			if erread != nil {
				errs = append(errs, fmt.Errorf("Skipping [FILE: %s] since it cannot be read (%s)", x, erread))
			} else {
				files = append(files, data)
			}
		}
	} else {
		data, erread := ioutil.ReadAll(stdin)
		if erread != nil {
			errs = append(errs, erread)
		} else {
			files = append(files, data)
		}
	}
	return files, errs
}

/*
Handling API errors so we can know why the request did not went as expetected.
The resquest will not be resend to prevent the user from being banned.
*/

func handleAPIError(error string) error {
	errorStr := make(map[string]string)
	errorStr["err_nothing_to_do"] = "No POST request was received by the create API"
	errorStr["err_author_numeric"] = "The paste author's alias should be alphanumeric"
	errorStr["err_save_error"] = "An error occurred while saving the paste"
	errorStr["err_spamguard_ipban"] = "Poster's IP address is banned"
	errorStr["err_spamguard_stealth"] = "The paste triggered the spam filter"
	errorStr["err_spamguard_noflood"] = "Poster is trying the flood"
	errorStr["err_spamguard_php"] = "Poster's IP address is listed as malicious"
	if err, ok := errorStr[error]; ok {
		return fmt.Errorf("API error: %s", err)
	}
	return fmt.Errorf("API error: Unknown [%s]", error)
}

func copyPaste(src []byte, opts *config) error {
	values := url.Values{
		"paste_data":     {string(src)},
		"paste_lang":     {*opts.lang},
		"api_submit":     {"true"},
		"mode":           {"json"},
		"paste_user":     {*opts.user},
		"paste_password": {*opts.pass},
	}
	if duration, err := time.ParseDuration(*opts.expire); err != nil {
		return err
	} else if secs := duration.Seconds(); secs >= 1 {
		values.Add("paste_expire", strconv.FormatFloat(secs, 'f', -1, 64))
	}
	if *opts.priv {
		values.Add("paste_private", "yes")
	}
	resp, erreq := http.PostForm(FPASTE_URL, values)
	if erreq != nil {
		return erreq
	}
	defer resp.Body.Close()
	type res struct {
		Id    string `json:"id"`
		Hash  string `json:"hash"`
		Error string `json:"error"`
	}
	type pasteUrls struct {
		Result res `json:"result"`
	}
	var m pasteUrls
	slice, err := ioutil.ReadAll(resp.Body)
	err = json.Unmarshal(slice, &m)
	if err != nil {
		return err
	}
	if m.Result.Error != "" {
		return handleAPIError(m.Result.Error)
	}
	fmt.Fprintf(os.Stdout, "%s/%s/%s\n", FPASTE_URL, m.Result.Id, m.Result.Hash)
	return nil
}
<commit_msg>adding default value for expire flag<commit_after>package main

import (
	getopt "code.google.com/p/getopt"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	http "net/http"
	url "net/url"
	"os"
	"strconv"
	"time"
)

const FPASTE_URL = "http://fpaste.org"

func main() {
	opts := initConfig(os.Args)
	if *opts.help {
		getopt.PrintUsage(os.Stdout)
	} else {
		/*Passing stdin for test mocking*/
		files, errs := handleArgs(os.Stdin, getopt.CommandLine)
		for _, file := range files {
			if len(file) != 0 {
				if err := copyPaste(file, opts); err != nil {
					errs = append(errs, err)
				}
			}
		}
		for _, err := range errs {
			log.Print(err)
		}
		if len(errs) > 0 {
			os.Exit(-1)
		}
	}
}

type config struct {
	help   *bool
	priv   *bool
	user   *string
	pass   *string
	lang   *string
	expire *string
}

func initConfig(args []string) *config {
	getopt.CommandLine = getopt.New()
	var flags config
	flags.help = getopt.BoolLong("help", 'h', "Display this help")
	flags.priv = getopt.BoolLong("private", 'P', "Private paste flag")
	flags.user = getopt.StringLong("user", 'u', "", "An alphanumeric username of the paste author")
	flags.pass = getopt.StringLong("pass", 'p', "", "Add a password")
	flags.lang = getopt.StringLong("lang", 'l', "Text", "The development language used")
	flags.expire = getopt.StringLong("expire", 'e', "0", "Seconds after which paste will be deleted from server")
	getopt.SetParameters("[FILE...]")
	getopt.CommandLine.Parse(args)
	return &flags
}

func handleArgs(stdin io.Reader, commandLine *getopt.Set) (files [][]byte, errs []error) {
	if commandLine.NArgs() > 0 {
		for _, x := range commandLine.Args() {
			file, err := os.Open(x)
			if err != nil {
				errs = append(errs, fmt.Errorf("Skipping [FILE: %s] since it cannot be opened (%s)", x, err))
				continue
			}
			defer file.Close()
			data, erread := ioutil.ReadAll(file)
			if erread != nil {
				errs = append(errs, fmt.Errorf("Skipping [FILE: %s] since it cannot be read (%s)", x, erread))
			} else {
				files = append(files, data)
			}
		}
	} else {
		data, erread := ioutil.ReadAll(stdin)
		if erread != nil {
			errs = append(errs, erread)
		} else {
			files = append(files, data)
		}
	}
	return files, errs
}

/*
Handling API errors so we can know why the request did not went as expetected.
The resquest will not be resend to prevent the user from being banned.
*/

func handleAPIError(error string) error {
	errorStr := make(map[string]string)
	errorStr["err_nothing_to_do"] = "No POST request was received by the create API"
	errorStr["err_author_numeric"] = "The paste author's alias should be alphanumeric"
	errorStr["err_save_error"] = "An error occurred while saving the paste"
	errorStr["err_spamguard_ipban"] = "Poster's IP address is banned"
	errorStr["err_spamguard_stealth"] = "The paste triggered the spam filter"
	errorStr["err_spamguard_noflood"] = "Poster is trying the flood"
	errorStr["err_spamguard_php"] = "Poster's IP address is listed as malicious"
	if err, ok := errorStr[error]; ok {
		return fmt.Errorf("API error: %s", err)
	}
	return fmt.Errorf("API error: Unknown [%s]", error)
}

func copyPaste(src []byte, opts *config) error {
	values := url.Values{
		"paste_data":     {string(src)},
		"paste_lang":     {*opts.lang},
		"api_submit":     {"true"},
		"mode":           {"json"},
		"paste_user":     {*opts.user},
		"paste_password": {*opts.pass},
	}
	if duration, err := time.ParseDuration(*opts.expire); err != nil {
		return err
	} else if secs := duration.Seconds(); secs >= 1 {
		values.Add("paste_expire", strconv.FormatFloat(secs, 'f', -1, 64))
	}
	if *opts.priv {
		values.Add("paste_private", "yes")
	}
	resp, erreq := http.PostForm(FPASTE_URL, values)
	if erreq != nil {
		return erreq
	}
	defer resp.Body.Close()
	type res struct {
		Id    string `json:"id"`
		Hash  string `json:"hash"`
		Error string `json:"error"`
	}
	type pasteUrls struct {
		Result res `json:"result"`
	}
	var m pasteUrls
	slice, err := ioutil.ReadAll(resp.Body)
	err = json.Unmarshal(slice, &m)
	if err != nil {
		return err
	}
	if m.Result.Error != "" {
		return handleAPIError(m.Result.Error)
	}
	fmt.Fprintf(os.Stdout, "%s/%s/%s\n", FPASTE_URL, m.Result.Id, m.Result.Hash)
	return nil
}
<|endoftext|>


<commit_before>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func sum(args ...int) int { return 0 }

var (
	_ = sum(1, 2, 3)
	_ = sum()
	_ = sum(1.0, 2.0)
	_ = sum(1.5)      // ERROR "integer"
	_ = sum("hello")  // ERROR "convert"
	_ = sum([]int{1}) // ERROR "slice literal.*as type int"
)

type T []T

func funny(args ...T) int { return 0 }

var (
	_ = funny(nil)
	_ = funny(nil, nil)
	_ = funny([]T{}) // ok because []T{} is a T; passes []T{[]T{}}
)
<commit_msg>Match gccgo error messages.<commit_after>// errchk $G -e $D/$F.go

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func sum(args ...int) int { return 0 }

var (
	_ = sum(1, 2, 3)
	_ = sum()
	_ = sum(1.0, 2.0)
	_ = sum(1.5)      // ERROR "integer"
	_ = sum("hello")  // ERROR "convert|incompatible"
	_ = sum([]int{1}) // ERROR "slice literal.*as type int|incompatible"
)

type T []T

func funny(args ...T) int { return 0 }

var (
	_ = funny(nil)
	_ = funny(nil, nil)
	_ = funny([]T{}) // ok because []T{} is a T; passes []T{[]T{}}
)
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"net"
)

// NodeManager receives and communicates with mgm Node processes
func NodeManager(listenPort string, hStatsLink chan<- HostStats, db Database, logger Logger) {

	ln, err := net.Listen("tcp", ":"+listenPort)
	if err != nil {
		logger.Fatal("MGM Node listener cannot start: ", err)
		return
	}
	logger.Info("Listening for mgmNode instances on :" + listenPort)

	go mgmConnectionAcceptor(ln, hStatsLink, db, logger)
}

func mgmConnectionAcceptor(listen net.Listener, hStatsLink chan<- HostStats, db Database, logger Logger) {
	for {
		conn, err := listen.Accept()
		if err != nil {
			logger.Error("Error accepting connection: ", err)
			continue
		}
		//validate connection, and identify host
		addr := conn.RemoteAddr()
		address := addr.(*net.TCPAddr).IP.String()
		host, err := db.GetHostByAddress(address)
		if err != nil {
			logger.Error("Error looking up mgm Node: ", err)
			continue
		}
		if host.Address != address {
			logger.Info("mgmNode connection from unregistered address: ", address)
			continue
		}
		logger.Info("MGM Node connection from: ", address)
		go mgmConnectionHandler(host.ID, conn, hStatsLink, logger)
	}
}

func mgmConnectionHandler(id uint, conn net.Conn, hStatsLink chan<- HostStats, logger Logger) {
	d := json.NewDecoder(conn)
	for {
		nmsg := NetworkMessage{}
		err := d.Decode(&nmsg)
		if err != nil {
			logger.Error("Error decoding mgmNode message: ", err)
			if err.Error() == "EOF" {
				return
			}
		}

		switch nmsg.MessageType {
		case "host_stats":
			hStats := nmsg.HStats
			hStats.ID = id
			hStatsLink <- hStats
		default:
			logger.Info("Received invalid message from an MGM node: ", nmsg.MessageType)
		}

	}
}
<commit_msg>fix logging line to print string content instead of string object<commit_after>package core

import (
	"encoding/json"
	"net"
)

// NodeManager receives and communicates with mgm Node processes
func NodeManager(listenPort string, hStatsLink chan<- HostStats, db Database, logger Logger) {

	ln, err := net.Listen("tcp", ":"+listenPort)
	if err != nil {
		logger.Fatal("MGM Node listener cannot start: ", err)
		return
	}
	logger.Info("Listening for mgmNode instances on :" + listenPort)

	go mgmConnectionAcceptor(ln, hStatsLink, db, logger)
}

func mgmConnectionAcceptor(listen net.Listener, hStatsLink chan<- HostStats, db Database, logger Logger) {
	for {
		conn, err := listen.Accept()
		if err != nil {
			logger.Error("Error accepting connection: ", err)
			continue
		}
		//validate connection, and identify host
		addr := conn.RemoteAddr()
		address := addr.(*net.TCPAddr).IP.String()
		host, err := db.GetHostByAddress(address)
		if err != nil {
			logger.Error("Error looking up mgm Node: ", err)
			continue
		}
		if host.Address != address {
			logger.Info("mgmNode connection from unregistered address: ", address)
			continue
		}
		logger.Info("MGM Node connection from: %v", address)
		go mgmConnectionHandler(host.ID, conn, hStatsLink, logger)
	}
}

func mgmConnectionHandler(id uint, conn net.Conn, hStatsLink chan<- HostStats, logger Logger) {
	d := json.NewDecoder(conn)
	for {
		nmsg := NetworkMessage{}
		err := d.Decode(&nmsg)
		if err != nil {
			logger.Error("Error decoding mgmNode message: ", err)
			if err.Error() == "EOF" {
				return
			}
		}

		switch nmsg.MessageType {
		case "host_stats":
			hStats := nmsg.HStats
			hStats.ID = id
			hStatsLink <- hStats
		default:
			logger.Info("Received invalid message from an MGM node: ", nmsg.MessageType)
		}

	}
}
<|endoftext|>


<commit_before>/*
	Package 'stage' handles assembling a release.

	Making a new release often takes a series of hitch commands --
	this matches how making a release often requires *several*
	large computations -- so all the intermediate staged states
	are serializable to disk.
*/
package stage

import (
	"bytes"
	stdjson "encoding/json"
	"io"
	"os"
	"path/filepath"

	"github.com/polydawn/refmt/json"

	"go.polydawn.net/hitch/api"
	"go.polydawn.net/hitch/core/db"
	. "go.polydawn.net/hitch/lib/errcat"
)

const DefaultPath = "_stage"

type Controller struct {
	dbctrl    *db.Controller
	stagePath string

	Catalog api.Catalog // catalog struct, sync'd with file.  always must have exactly one release entry.
}

/*
	Create a new empty release staging state.  Makes a dir, and creates the sigil file.
*/
func Create(
	dbctrl *db.Controller, stagePath string,
	catalogName api.CatalogName, releaseName api.ReleaseName,
) (*Controller, error) {
	err := os.MkdirAll(filepath.Join(dbctrl.BasePath, stagePath), 0755)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	f, err := os.OpenFile(filepath.Join(dbctrl.BasePath, stagePath, "stage.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	defer f.Close()
	stageCtrl := &Controller{
		dbctrl:    dbctrl,
		stagePath: stagePath,

		Catalog: api.Catalog{
			Name: catalogName,
			Releases: []api.ReleaseEntry{
				{Name: releaseName},
			},
		},
	}
	return stageCtrl, stageCtrl.flush(f)
}

func (stageCtrl *Controller) Save() error {
	f, err := os.OpenFile(filepath.Join(stageCtrl.dbctrl.BasePath, stageCtrl.stagePath, "stage.json"), os.O_WRONLY|os.O_TRUNC, 0)
	if err != nil {
		return Errorw(ErrIO, err)
	}
	defer f.Close()
	return stageCtrl.flush(f)
}

func (stageCtrl *Controller) flush(w io.Writer) error {
	msg, err := json.MarshalAtlased(stageCtrl.Catalog, api.Atlas)
	if err != nil {
		panic(err) // marshalling into a buffer shouldn't fail!
	}
	var buf bytes.Buffer
	stdjson.Indent(&buf, msg, "", "\t")
	_, err = buf.WriteTo(w)
	return Errorw(ErrIO, err)
}

func Load(dbctrl *db.Controller, stagePath string) (*Controller, error) {
	f, err := os.OpenFile(filepath.Join(dbctrl.BasePath, stagePath, "stage.json"), os.O_RDONLY, 0)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	defer f.Close()

	stageCtrl := &Controller{
		dbctrl:    dbctrl,
		stagePath: stagePath,
	}
	return stageCtrl, stageCtrl.load(f)
}

func (stageCtrl *Controller) load(r io.Reader) error {
	err := json.NewUnmarshallerAtlased(r, api.Atlas).
		Unmarshal(&stageCtrl.Catalog)
	return Errorw(ErrStorageCorrupt, err)
}
<commit_msg>stage: demo drew attn to lack of trailing break.<commit_after>/*
	Package 'stage' handles assembling a release.

	Making a new release often takes a series of hitch commands --
	this matches how making a release often requires *several*
	large computations -- so all the intermediate staged states
	are serializable to disk.
*/
package stage

import (
	"bytes"
	stdjson "encoding/json"
	"io"
	"os"
	"path/filepath"

	"github.com/polydawn/refmt/json"

	"go.polydawn.net/hitch/api"
	"go.polydawn.net/hitch/core/db"
	. "go.polydawn.net/hitch/lib/errcat"
)

const DefaultPath = "_stage"

type Controller struct {
	dbctrl    *db.Controller
	stagePath string

	Catalog api.Catalog // catalog struct, sync'd with file.  always must have exactly one release entry.
}

/*
	Create a new empty release staging state.  Makes a dir, and creates the sigil file.
*/
func Create(
	dbctrl *db.Controller, stagePath string,
	catalogName api.CatalogName, releaseName api.ReleaseName,
) (*Controller, error) {
	err := os.MkdirAll(filepath.Join(dbctrl.BasePath, stagePath), 0755)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	f, err := os.OpenFile(filepath.Join(dbctrl.BasePath, stagePath, "stage.json"), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	defer f.Close()
	stageCtrl := &Controller{
		dbctrl:    dbctrl,
		stagePath: stagePath,

		Catalog: api.Catalog{
			Name: catalogName,
			Releases: []api.ReleaseEntry{
				{Name: releaseName},
			},
		},
	}
	return stageCtrl, stageCtrl.flush(f)
}

func (stageCtrl *Controller) Save() error {
	f, err := os.OpenFile(filepath.Join(stageCtrl.dbctrl.BasePath, stageCtrl.stagePath, "stage.json"), os.O_WRONLY|os.O_TRUNC, 0)
	if err != nil {
		return Errorw(ErrIO, err)
	}
	defer f.Close()
	return stageCtrl.flush(f)
}

func (stageCtrl *Controller) flush(w io.Writer) error {
	msg, err := json.MarshalAtlased(stageCtrl.Catalog, api.Atlas)
	if err != nil {
		panic(err) // marshalling into a buffer shouldn't fail!
	}
	var buf bytes.Buffer
	stdjson.Indent(&buf, msg, "", "\t")
	buf.WriteString("\n")
	_, err = buf.WriteTo(w)
	return Errorw(ErrIO, err)
}

func Load(dbctrl *db.Controller, stagePath string) (*Controller, error) {
	f, err := os.OpenFile(filepath.Join(dbctrl.BasePath, stagePath, "stage.json"), os.O_RDONLY, 0)
	if err != nil {
		return nil, Errorw(ErrIO, err)
	}
	defer f.Close()

	stageCtrl := &Controller{
		dbctrl:    dbctrl,
		stagePath: stagePath,
	}
	return stageCtrl, stageCtrl.load(f)
}

func (stageCtrl *Controller) load(r io.Reader) error {
	err := json.NewUnmarshallerAtlased(r, api.Atlas).
		Unmarshal(&stageCtrl.Catalog)
	return Errorw(ErrStorageCorrupt, err)
}
<|endoftext|>


<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"
)

type metadata map[string]interface{}

// filenameURL returns an IPFS reference including a filename, if available.
// e.g. /ipfs/<parent_hash>/my_file.jpg instead of /ipfs/<file_hash>/
// This helps Tika with file type detection.
func (i *Indexable) getFilenameURL() (path string) {
	if i.Name != "" && i.ParentHash != "" {
		return fmt.Sprintf("/ipfs/%s/%s", i.ParentHash, i.Name)
	}

	// No name & parent hash available
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

func (i *Indexable) retryingGet(url string) (resp *http.Response, err error) {
	client := http.Client{
		Timeout: i.Config.IpfsTikaTimeout,
	}

	tryAgain := true
	for tryAgain {
		resp, err = client.Get(url)

		tryAgain, err = i.handleURLError(err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// getTika requests IPFS path from IPFS-TIKA and writes returned metadata
func (i *Indexable) getTika(m *metadata) error {
	resp, err := i.retryingGet(i.Config.IpfsTikaURL + i.getFilenameURL())

	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("undesired status '%s' from ipfs-tika", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
		return err
	}

	return err
}

// getMatadata sets metdata for file with args or returns error
func (i *Indexable) getMetadata(m *metadata) error {
	var err error

	if i.Args.Size > 0 {
		if i.Args.Size > i.Config.MetadataMaxSize {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s too large, not indexing (for now)", i)
		}

		err = i.getTika(m)
		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
		   for raw_url := range metadata.urls {
		       url, err := URL.Parse(raw_url)

		       if err != nil {
		           return err
		       }

		       if strings.HasPrefix(url.Path, "/ipfs/") {
		           // Found IPFS link!
		           args := crawlerArgs{
		               Hash:       link.Hash,
		               Name:       link.Name,
		               Size:       link.Size,
		               ParentHash: hash,
		           }

		       }
		   }
		*/
	}

	return nil
}
<commit_msg>Explicitly log metadata fetching.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"
)

type metadata map[string]interface{}

// filenameURL returns an IPFS reference including a filename, if available.
// e.g. /ipfs/<parent_hash>/my_file.jpg instead of /ipfs/<file_hash>/
// This helps Tika with file type detection.
func (i *Indexable) getFilenameURL() (path string) {
	if i.Name != "" && i.ParentHash != "" {
		return fmt.Sprintf("/ipfs/%s/%s", i.ParentHash, i.Name)
	}

	// No name & parent hash available
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

func (i *Indexable) retryingGet(url string) (resp *http.Response, err error) {
	client := http.Client{
		Timeout: i.Config.IpfsTikaTimeout,
	}

	tryAgain := true
	for tryAgain {
		log.Printf("Fetching metadata from '%s'", url)
		resp, err = client.Get(url)

		tryAgain, err = i.handleURLError(err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// getTika requests IPFS path from IPFS-TIKA and writes returned metadata
func (i *Indexable) getTika(m *metadata) error {
	resp, err := i.retryingGet(i.Config.IpfsTikaURL + i.getFilenameURL())

	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("undesired status '%s' from ipfs-tika", resp.Status)
	}

	// Parse resulting JSON
	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
		return err
	}

	return err
}

// getMatadata sets metdata for file with args or returns error
func (i *Indexable) getMetadata(m *metadata) error {
	var err error

	if i.Args.Size > 0 {
		if i.Args.Size > i.Config.MetadataMaxSize {
			// Fail hard for really large files, for now
			return fmt.Errorf("%s too large, not indexing (for now)", i)
		}

		err = i.getTika(m)
		if err != nil {
			return err
		}

		// Check for IPFS links in content
		/*
		   for raw_url := range metadata.urls {
		       url, err := URL.Parse(raw_url)

		       if err != nil {
		           return err
		       }

		       if strings.HasPrefix(url.Path, "/ipfs/") {
		           // Found IPFS link!
		           args := crawlerArgs{
		               Hash:       link.Hash,
		               Name:       link.Name,
		               Size:       link.Size,
		               ParentHash: hash,
		           }

		       }
		   }
		*/
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package record

import (
	"strings"

	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	"github.com/skygeario/skygear-server/pkg/server/skyerr"
)

// SortOrder denotes an the order of Records returned from a Query.
type SortOrder int

// A list of SordOrder, their meaning is self descriptive.
const (
	Ascending SortOrder = iota
	Descending
	Asc  = Ascending
	Desc = Descending
)

// Sort specifies the order of a collection of Records returned from a Query.
//
// Record order can be sorted w.r.t. a record field or a value returned
// from a predefined function.
type Sort struct {
	Expression Expression
	Order      SortOrder
}

// Accept implements the Visitor pattern.
func (sort Sort) Accept(visitor Visitor) {
	if v, ok := visitor.(SortVisitor); ok {
		v.VisitSort(sort)
		defer v.EndVisitSort(sort)
	}
	if v, ok := visitor.(ExpressionVisitor); ok {
		sort.Expression.Accept(v)
	}
}

// Operator denotes how the result of a predicate is determined from
// its subpredicates or subexpressions.
//go:generate stringer -type=Operator
type Operator int

// A list of Operator.
const (
	And Operator = iota + 1
	Or
	Not
	Equal
	GreaterThan
	LessThan
	GreaterThanOrEqual
	LessThanOrEqual
	NotEqual
	Like
	ILike
	In
	Functional
)

// IsCompound checks whether the Operator is a compound operator, meaning the
// operator combine the results of other subpredicates.
func (op Operator) IsCompound() bool {
	switch op {
	default:
		return false
	case And, Or, Not:
		return true
	}
}

// IsBinary checks whether the Operator determines the result of a predicate
// by comparing two subexpressions.
func (op Operator) IsBinary() bool {
	switch op {
	default:
		return false
	case Equal, GreaterThan, LessThan, GreaterThanOrEqual, LessThanOrEqual, NotEqual, Like, ILike, In:
		return true
	}
}

// IsCommutative checks whether expressions on both side of the Operator
// can be swapped.
func (op Operator) IsCommutative() bool {
	switch op {
	default:
		return false
	case Equal, NotEqual:
		return true
	}
}

// ExpressionType is the type of an Expression.
type ExpressionType int

// A list of ExpressionTypes.
const (
	Literal ExpressionType = iota + 1
	KeyPath
	Function
)

// An Expression represents value to be compared against.
type Expression struct {
	Type  ExpressionType
	Value interface{}
}

func (expr Expression) IsEmpty() bool {
	return expr.Type == 0 && expr.Value == nil
}

func (expr Expression) IsKeyPath() bool {
	return expr.Type == KeyPath
}

func (expr Expression) IsLiteralString() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(string)
	return ok
}

func (expr Expression) IsLiteralArray() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.([]interface{})
	return ok
}

func (expr Expression) IsLiteralMap() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(map[string]interface{})
	return ok
}

func (expr Expression) IsLiteralNull() bool {
	if expr.Type != Literal {
		return false
	}

	return expr.Value == nil
}

func (expr Expression) KeyPathComponents() []string {
	if expr.Type != KeyPath {
		panic("expression is not a keypath")
	}

	return strings.Split(expr.Value.(string), ".")
}

// Accept implements the Visitor pattern.
func (expr Expression) Accept(visitor Visitor) {
	if v, ok := visitor.(ExpressionVisitor); ok {
		v.VisitExpression(expr)
		v.EndVisitExpression(expr)
	}
}

// Predicate is a representation of used in query for filtering records.
type Predicate struct {
	Operator Operator
	Children []interface{}
}

func (p Predicate) IsEmpty() bool {
	return p.Operator == 0 || p.Children == nil
}

// Validate returns an Error if a Predicate is invalid.
//
// If a Predicate is validated without error, nil is returned.
func (p Predicate) Validate() skyerr.Error {
	return p.validate(nil)
}

// validate is an internal version of the exported Validate() function.
//
// Additional information is passed as parameter to check the context
// in which the predicate is specified.
func (p Predicate) validate(parentPredicate *Predicate) skyerr.Error {
	if p.Operator.IsBinary() && len(p.Children) != 2 {
		return skyerr.NewErrorf(skyerr.RecordQueryInvalid,
			"binary predicate must have 2 operands, got %d", len(p.Children))
	}
	if p.Operator == Functional && len(p.Children) != 1 {
		return skyerr.NewErrorf(skyerr.RecordQueryInvalid,
			"functional predicate must have 1 operand, got %d", len(p.Children))
	}

	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				return skyerr.NewError(skyerr.RecordQueryInvalid,
					"children of compound predicate must be a predicate")
			}

			if err := predicate.validate(&p); err != nil {
				return err
			}
		}
	} else {
		for _, child := range p.Children {
			_, ok := child.(Expression)
			if !ok {
				return skyerr.NewError(skyerr.RecordQueryInvalid,
					"children of simple predicate must be an expression")
			}
		}
	}

	switch p.Operator {
	case Functional:
		return p.validateFunctionalPredicate(parentPredicate)
	case Equal:
		return p.validateEqualPredicate(parentPredicate)
	}
	return nil
}

func (p Predicate) validateFunctionalPredicate(parentPredicate *Predicate) skyerr.Error {
	expr := p.Children[0].(Expression)
	if expr.Type != Function {
		return skyerr.NewError(skyerr.RecordQueryInvalid,
			`functional predicate must contain functional expression`)
	}

	switch f := expr.Value.(type) {
	case UserRelationFunc:
		if f.RelationName != "_friend" && f.RelationName != "_follow" {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`user relation predicate with "%d" relation is not supported`,
				f.RelationName)
		}
	default:
		return skyerr.NewError(skyerr.NotSupported,
			`unsupported function for functional predicate`)
	}
	return nil
}

func (p Predicate) validateEqualPredicate(parentPredicate *Predicate) skyerr.Error {
	lhs := p.Children[0].(Expression)
	rhs := p.Children[1].(Expression)

	if lhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			lhs.Value)
	} else if lhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			lhs.Value)
	} else if rhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			rhs.Value)
	} else if rhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			rhs.Value)
	}
	return nil
}

// GetSubPredicates returns Predicate.Children as []Predicate.
//
// This method is only valid when Operator is either And, Or and Not. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetSubPredicates() (ps []Predicate) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Predicate))
	}
	return
}

// GetExpressions returns Predicate.Children as []Expression.
//
// This method is only valid when Operator is binary operator. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetExpressions() (ps []Expression) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Expression))
	}
	return
}

// Accept implements the Visitor pattern.
func (p Predicate) Accept(visitor Visitor) {
	if v, ok := visitor.(PredicateVisitor); ok {
		v.VisitPredicate(p)
		defer v.EndVisitPredicate(p)
	}
	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				panic("children of compound predicate must be a predicate")
			}

			predicate.Accept(visitor)
		}
	} else {
		if v, ok := visitor.(ExpressionVisitor); ok {
			for _, child := range p.Children {
				expr, ok := child.(Expression)
				if !ok {
					panic("children of simple predicate must be an expression")
				}

				expr.Accept(v)
			}
		}
	}
}

// Query specifies the type, predicate and sorting order of Database
// query.
type Query struct {
	Type         string
	Predicate    Predicate
	Sorts        []Sort
	ComputedKeys map[string]Expression
	DesiredKeys  []string
	GetCount     bool
	Limit        *uint64
	Offset       uint64
}

// Accept implements the Visitor pattern.
func (q Query) Accept(visitor Visitor) {
	if v, ok := visitor.(QueryVisitor); ok {
		v.VisitQuery(q)
		defer v.EndVisitQuery(q)
	}

	if v, ok := visitor.(PredicateVisitor); ok {
		q.Predicate.Accept(v)
	}

	if v, ok := visitor.(SortVisitor); ok {
		for _, sort := range q.Sorts {
			sort.Accept(v)
		}
	}

	if v, ok := visitor.(ExpressionVisitor); ok {
		for _, expr := range q.ComputedKeys {
			expr.Accept(v)
		}
	}
}

// AccessControlOptions provide access control options to query.
//
// The following fields are generated from the server side, rather
// than supplied from the client side.
type AccessControlOptions struct {
	ViewAsUser          *authinfo.AuthInfo
	BypassAccessControl bool
}

// Func is a marker interface to denote a type being a function in skydb.
//
// skydb's function receives zero or more arguments and returns a DataType
// as a result. Result data type is currently omitted in this interface since
// skygear doesn't use it internally yet. In the future it can be utilized to
// provide more extensive type checking at handler level.
type Func interface {
	Args() []interface{}
	DataType() DataType
}

// KeyPathFunc is a marker interface to denote a func that
// references certain key paths.
type KeyPathFunc interface {
	// Returns a list of key paths that is referenced by this function.
	ReferencedKeyPaths() []string
}

// DistanceFunc represents a function that calculates distance between
// a user supplied location and a Record's field
type DistanceFunc struct {
	Field    string
	Location Location
}

// Args implements the Func interface
func (f DistanceFunc) Args() []interface{} {
	return []interface{}{f.Field, f.Location}
}

func (f DistanceFunc) DataType() DataType {
	return TypeNumber
}

// ReferencedKeyPaths implements the KeyPathFunc interface.
func (f DistanceFunc) ReferencedKeyPaths() []string {
	return []string{f.Field}
}

// CountFunc represents a function that count number of rows matching
// a query
type CountFunc struct {
	OverallRecords bool
}

// Args implements the Func interface
func (f CountFunc) Args() []interface{} {
	return []interface{}{}
}

func (f CountFunc) DataType() DataType {
	return TypeNumber
}

// UserRelationFunc represents a function that is used to evaulate
// whether a record satisfy certain user-based relation
type UserRelationFunc struct {
	KeyPath           string
	RelationName      string
	RelationDirection string
	User              string
}

// Args implements the Func interface
func (f UserRelationFunc) Args() []interface{} {
	return []interface{}{}
}

func (f UserRelationFunc) DataType() DataType {
	return TypeBoolean
}

// ReferencedKeyPaths implements the KeyPathFunc interface.
func (f UserRelationFunc) ReferencedKeyPaths() []string {
	return []string{f.KeyPath}
}

// Visitor is a marker interface
type Visitor interface{}

// FullQueryVisitor is a marker interface for all query-related visitors
type FullQueryVisitor interface {
	QueryVisitor
	PredicateVisitor
	SortVisitor
	ExpressionVisitor
}

// QueryVisitor is an interface that implements the Visitor pattern for
// the Query struct.
type QueryVisitor interface {
	VisitQuery(Query)
	EndVisitQuery(Query)
}

// PredicateVisitor is an interface that implements the Visitor pattern for
// the Predicate struct.
type PredicateVisitor interface {
	VisitPredicate(Predicate)
	EndVisitPredicate(Predicate)
}

// SortVisitor is an interface that implements the Visitor pattern for
// the Sort struct.
type SortVisitor interface {
	VisitSort(Sort)
	EndVisitSort(Sort)
}

// ExpressionVisitor is an interface that implements the Visitor pattern for
// the Expression struct.
type ExpressionVisitor interface {
	VisitExpression(Expression)
	EndVisitExpression(Expression)
}
<commit_msg>Fix wrong error format string in record/query.go<commit_after>// Copyright 2015-present Oursky Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package record

import (
	"strings"

	"github.com/skygeario/skygear-server/pkg/core/auth/authinfo"
	"github.com/skygeario/skygear-server/pkg/server/skyerr"
)

// SortOrder denotes an the order of Records returned from a Query.
type SortOrder int

// A list of SordOrder, their meaning is self descriptive.
const (
	Ascending SortOrder = iota
	Descending
	Asc  = Ascending
	Desc = Descending
)

// Sort specifies the order of a collection of Records returned from a Query.
//
// Record order can be sorted w.r.t. a record field or a value returned
// from a predefined function.
type Sort struct {
	Expression Expression
	Order      SortOrder
}

// Accept implements the Visitor pattern.
func (sort Sort) Accept(visitor Visitor) {
	if v, ok := visitor.(SortVisitor); ok {
		v.VisitSort(sort)
		defer v.EndVisitSort(sort)
	}
	if v, ok := visitor.(ExpressionVisitor); ok {
		sort.Expression.Accept(v)
	}
}

// Operator denotes how the result of a predicate is determined from
// its subpredicates or subexpressions.
//go:generate stringer -type=Operator
type Operator int

// A list of Operator.
const (
	And Operator = iota + 1
	Or
	Not
	Equal
	GreaterThan
	LessThan
	GreaterThanOrEqual
	LessThanOrEqual
	NotEqual
	Like
	ILike
	In
	Functional
)

// IsCompound checks whether the Operator is a compound operator, meaning the
// operator combine the results of other subpredicates.
func (op Operator) IsCompound() bool {
	switch op {
	default:
		return false
	case And, Or, Not:
		return true
	}
}

// IsBinary checks whether the Operator determines the result of a predicate
// by comparing two subexpressions.
func (op Operator) IsBinary() bool {
	switch op {
	default:
		return false
	case Equal, GreaterThan, LessThan, GreaterThanOrEqual, LessThanOrEqual, NotEqual, Like, ILike, In:
		return true
	}
}

// IsCommutative checks whether expressions on both side of the Operator
// can be swapped.
func (op Operator) IsCommutative() bool {
	switch op {
	default:
		return false
	case Equal, NotEqual:
		return true
	}
}

// ExpressionType is the type of an Expression.
type ExpressionType int

// A list of ExpressionTypes.
const (
	Literal ExpressionType = iota + 1
	KeyPath
	Function
)

// An Expression represents value to be compared against.
type Expression struct {
	Type  ExpressionType
	Value interface{}
}

func (expr Expression) IsEmpty() bool {
	return expr.Type == 0 && expr.Value == nil
}

func (expr Expression) IsKeyPath() bool {
	return expr.Type == KeyPath
}

func (expr Expression) IsLiteralString() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(string)
	return ok
}

func (expr Expression) IsLiteralArray() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.([]interface{})
	return ok
}

func (expr Expression) IsLiteralMap() bool {
	if expr.Type != Literal {
		return false
	}

	_, ok := expr.Value.(map[string]interface{})
	return ok
}

func (expr Expression) IsLiteralNull() bool {
	if expr.Type != Literal {
		return false
	}

	return expr.Value == nil
}

func (expr Expression) KeyPathComponents() []string {
	if expr.Type != KeyPath {
		panic("expression is not a keypath")
	}

	return strings.Split(expr.Value.(string), ".")
}

// Accept implements the Visitor pattern.
func (expr Expression) Accept(visitor Visitor) {
	if v, ok := visitor.(ExpressionVisitor); ok {
		v.VisitExpression(expr)
		v.EndVisitExpression(expr)
	}
}

// Predicate is a representation of used in query for filtering records.
type Predicate struct {
	Operator Operator
	Children []interface{}
}

func (p Predicate) IsEmpty() bool {
	return p.Operator == 0 || p.Children == nil
}

// Validate returns an Error if a Predicate is invalid.
//
// If a Predicate is validated without error, nil is returned.
func (p Predicate) Validate() skyerr.Error {
	return p.validate(nil)
}

// validate is an internal version of the exported Validate() function.
//
// Additional information is passed as parameter to check the context
// in which the predicate is specified.
func (p Predicate) validate(parentPredicate *Predicate) skyerr.Error {
	if p.Operator.IsBinary() && len(p.Children) != 2 {
		return skyerr.NewErrorf(skyerr.RecordQueryInvalid,
			"binary predicate must have 2 operands, got %d", len(p.Children))
	}
	if p.Operator == Functional && len(p.Children) != 1 {
		return skyerr.NewErrorf(skyerr.RecordQueryInvalid,
			"functional predicate must have 1 operand, got %d", len(p.Children))
	}

	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				return skyerr.NewError(skyerr.RecordQueryInvalid,
					"children of compound predicate must be a predicate")
			}

			if err := predicate.validate(&p); err != nil {
				return err
			}
		}
	} else {
		for _, child := range p.Children {
			_, ok := child.(Expression)
			if !ok {
				return skyerr.NewError(skyerr.RecordQueryInvalid,
					"children of simple predicate must be an expression")
			}
		}
	}

	switch p.Operator {
	case Functional:
		return p.validateFunctionalPredicate(parentPredicate)
	case Equal:
		return p.validateEqualPredicate(parentPredicate)
	}
	return nil
}

func (p Predicate) validateFunctionalPredicate(parentPredicate *Predicate) skyerr.Error {
	expr := p.Children[0].(Expression)
	if expr.Type != Function {
		return skyerr.NewError(skyerr.RecordQueryInvalid,
			`functional predicate must contain functional expression`)
	}

	switch f := expr.Value.(type) {
	case UserRelationFunc:
		if f.RelationName != "_friend" && f.RelationName != "_follow" {
			return skyerr.NewErrorf(skyerr.NotSupported,
				`user relation predicate with "%s" relation is not supported`,
				f.RelationName)
		}
	default:
		return skyerr.NewError(skyerr.NotSupported,
			`unsupported function for functional predicate`)
	}
	return nil
}

func (p Predicate) validateEqualPredicate(parentPredicate *Predicate) skyerr.Error {
	lhs := p.Children[0].(Expression)
	rhs := p.Children[1].(Expression)

	if lhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			lhs.Value)
	} else if lhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			lhs.Value)
	} else if rhs.IsLiteralMap() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of map "%v" is not supported`,
			rhs.Value)
	} else if rhs.IsLiteralArray() {
		return skyerr.NewErrorf(skyerr.NotSupported,
			`equal comparison of array "%v" is not supported`,
			rhs.Value)
	}
	return nil
}

// GetSubPredicates returns Predicate.Children as []Predicate.
//
// This method is only valid when Operator is either And, Or and Not. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetSubPredicates() (ps []Predicate) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Predicate))
	}
	return
}

// GetExpressions returns Predicate.Children as []Expression.
//
// This method is only valid when Operator is binary operator. Caller
// is responsible to check for this preconditions. Otherwise the method
// will panic.
func (p Predicate) GetExpressions() (ps []Expression) {
	for _, childPred := range p.Children {
		ps = append(ps, childPred.(Expression))
	}
	return
}

// Accept implements the Visitor pattern.
func (p Predicate) Accept(visitor Visitor) {
	if v, ok := visitor.(PredicateVisitor); ok {
		v.VisitPredicate(p)
		defer v.EndVisitPredicate(p)
	}
	if p.Operator.IsCompound() {
		for _, child := range p.Children {
			predicate, ok := child.(Predicate)
			if !ok {
				panic("children of compound predicate must be a predicate")
			}

			predicate.Accept(visitor)
		}
	} else {
		if v, ok := visitor.(ExpressionVisitor); ok {
			for _, child := range p.Children {
				expr, ok := child.(Expression)
				if !ok {
					panic("children of simple predicate must be an expression")
				}

				expr.Accept(v)
			}
		}
	}
}

// Query specifies the type, predicate and sorting order of Database
// query.
type Query struct {
	Type         string
	Predicate    Predicate
	Sorts        []Sort
	ComputedKeys map[string]Expression
	DesiredKeys  []string
	GetCount     bool
	Limit        *uint64
	Offset       uint64
}

// Accept implements the Visitor pattern.
func (q Query) Accept(visitor Visitor) {
	if v, ok := visitor.(QueryVisitor); ok {
		v.VisitQuery(q)
		defer v.EndVisitQuery(q)
	}

	if v, ok := visitor.(PredicateVisitor); ok {
		q.Predicate.Accept(v)
	}

	if v, ok := visitor.(SortVisitor); ok {
		for _, sort := range q.Sorts {
			sort.Accept(v)
		}
	}

	if v, ok := visitor.(ExpressionVisitor); ok {
		for _, expr := range q.ComputedKeys {
			expr.Accept(v)
		}
	}
}

// AccessControlOptions provide access control options to query.
//
// The following fields are generated from the server side, rather
// than supplied from the client side.
type AccessControlOptions struct {
	ViewAsUser          *authinfo.AuthInfo
	BypassAccessControl bool
}

// Func is a marker interface to denote a type being a function in skydb.
//
// skydb's function receives zero or more arguments and returns a DataType
// as a result. Result data type is currently omitted in this interface since
// skygear doesn't use it internally yet. In the future it can be utilized to
// provide more extensive type checking at handler level.
type Func interface {
	Args() []interface{}
	DataType() DataType
}

// KeyPathFunc is a marker interface to denote a func that
// references certain key paths.
type KeyPathFunc interface {
	// Returns a list of key paths that is referenced by this function.
	ReferencedKeyPaths() []string
}

// DistanceFunc represents a function that calculates distance between
// a user supplied location and a Record's field
type DistanceFunc struct {
	Field    string
	Location Location
}

// Args implements the Func interface
func (f DistanceFunc) Args() []interface{} {
	return []interface{}{f.Field, f.Location}
}

func (f DistanceFunc) DataType() DataType {
	return TypeNumber
}

// ReferencedKeyPaths implements the KeyPathFunc interface.
func (f DistanceFunc) ReferencedKeyPaths() []string {
	return []string{f.Field}
}

// CountFunc represents a function that count number of rows matching
// a query
type CountFunc struct {
	OverallRecords bool
}

// Args implements the Func interface
func (f CountFunc) Args() []interface{} {
	return []interface{}{}
}

func (f CountFunc) DataType() DataType {
	return TypeNumber
}

// UserRelationFunc represents a function that is used to evaulate
// whether a record satisfy certain user-based relation
type UserRelationFunc struct {
	KeyPath           string
	RelationName      string
	RelationDirection string
	User              string
}

// Args implements the Func interface
func (f UserRelationFunc) Args() []interface{} {
	return []interface{}{}
}

func (f UserRelationFunc) DataType() DataType {
	return TypeBoolean
}

// ReferencedKeyPaths implements the KeyPathFunc interface.
func (f UserRelationFunc) ReferencedKeyPaths() []string {
	return []string{f.KeyPath}
}

// Visitor is a marker interface
type Visitor interface{}

// FullQueryVisitor is a marker interface for all query-related visitors
type FullQueryVisitor interface {
	QueryVisitor
	PredicateVisitor
	SortVisitor
	ExpressionVisitor
}

// QueryVisitor is an interface that implements the Visitor pattern for
// the Query struct.
type QueryVisitor interface {
	VisitQuery(Query)
	EndVisitQuery(Query)
}

// PredicateVisitor is an interface that implements the Visitor pattern for
// the Predicate struct.
type PredicateVisitor interface {
	VisitPredicate(Predicate)
	EndVisitPredicate(Predicate)
}

// SortVisitor is an interface that implements the Visitor pattern for
// the Sort struct.
type SortVisitor interface {
	VisitSort(Sort)
	EndVisitSort(Sort)
}

// ExpressionVisitor is an interface that implements the Visitor pattern for
// the Expression struct.
type ExpressionVisitor interface {
	VisitExpression(Expression)
	EndVisitExpression(Expression)
}
<|endoftext|>


<commit_before>package transferarchiver

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	slashpath "path"

	"github.com/pkg/errors"
)

var (
	//TarArchiverKey configures the one object key returned by the tar Archiver
	TarArchiverKey = "archive.tar"

	//TarArchiverPathSeparator standardizes the header path for cross platform (un)archiving
	TarArchiverPathSeparator = "/"

	//ErrNoSuchDirectory is returned when an archiver expected a directory to exist
	ErrNoSuchDirectory = errors.New("directory doesn't exist")

	//ErrEmptyDirectory is returned when the archiver expected the directory to not be empty
	ErrEmptyDirectory = errors.New("directory is empty")
)

//TarArchiver will archive a directory into a single tar file
type TarArchiver struct {
	keyPrefix string
}

//NewTarArchiver will setup the tar archiver
func NewTarArchiver(opts ArchiverOptions) (a *TarArchiver, err error) {
	a = &TarArchiver{keyPrefix: opts.TarArchiverKeyPrefix}

	if a.keyPrefix != "" && !strings.HasSuffix(a.keyPrefix, "/") {
		return nil, errors.Errorf("archiver key prefix must end with a forward slash")
	}

	return a, nil
}

//tempFile will setup a temproary file that can easily be cleaned
func (a *TarArchiver) tempFile() (f *os.File, clean func(), err error) {
	f, err = ioutil.TempFile("", "tar_archiver_")
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file")
	}

	return f, func() {
		_ = f.Close()
		_ = os.Remove(f.Name())
	}, nil
}

func (a *TarArchiver) checkTargetDir(path string) error {
	dir, err := os.Open(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return errors.Wrap(err, "failed to open directory")
		}

		err = os.Mkdir(path, 0777) //@TODO decide on permissions before umask
		if err != nil {
			return errors.Wrap(err, "failed to create directory")
		}

		dir, err = os.Open(path)
		if err != nil {
			return errors.Wrap(err, "failed open created directory")
		}
	}

	fis, err := dir.Readdirnames(1)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "failed to read directory")
	}

	if len(fis) > 0 {
		return errors.New("directory is not empty")
	}

	return nil
}

//Index calls 'fn' for all object keys that are part of the archive
func (a *TarArchiver) Index(fn func(k string) error) error {
	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey))
}

//@TODO do we want to expose this through the interface?
func (a *TarArchiver) indexFS(path string, fn func(p string, fi os.FileInfo, err error) error) error {
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if fi == nil || path == p {
			return nil //this is triggered when a directory doesn't have an executable bit
		}
		if err != nil {
			return err
		}

		return fn(p, fi, nil)
	}); err != nil {
		return err
	}

	return nil
}

//Archive will archive a directory at 'path' into readable objects 'r' and calls 'fn' for each
func (a *TarArchiver) Archive(path string, rep Reporter, fn func(k string, r io.ReadSeeker, nbytes int64) error) (err error) {
	_, err = os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return ErrNoSuchDirectory
		}

		return err
	}

	var totalToTar int64
	i := 0
	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		i++
		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}
		totalToTar += fi.Size()
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to index filesystem")
	}

	if i <= 1 {
		// return errors.New("cannot archive empty directory")
		return ErrEmptyDirectory
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	inc := rep.StartArchivingProgress(tmpf.Name(), totalToTar)

	defer clean()
	tw := tar.NewWriter(tmpf)
	defer tw.Close()

	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		rel, err := filepath.Rel(path, p)
		if err != nil {
			return errors.Wrap(err, "failed to determine relative path")
		}

		//write header with a filename that standardizes the Separator
		path := strings.Split(rel, string(filepath.Separator))
		hdr, err := tar.FileInfoHeader(fi, "") //@TODO find out how we handle symlinks
		if err != nil {
			return errors.Wrap(err, "failed to convert file info to tar header")
		}

		hdr.Name = strings.Join(path, TarArchiverPathSeparator)
		if err = tw.WriteHeader(hdr); err != nil {
			return errors.Wrap(err, "failed to write tar header")
		}

		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		// open files for taring
		f, err := os.Open(p)
		defer f.Close()
		if err != nil {
			return errors.Wrap(err, "failed to open file for archiving")
		}

		// copy file data into tar writer
		var n int64
		if n, err = io.Copy(tw, f); err != nil {
			return errors.Wrap(err, "failed to copy file content to archive")
		}
		inc(n)
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to perform filesystem walk")
	}
	err = tw.Flush()
	if err != nil {
		return errors.Wrap(err, "failed to flush tar writer to disk")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to beginning of file")
	}

	//stop progress reporting, we're done
	rep.StopArchivingProgress()
	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat the temporary file")
	}

	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf, fi.Size())
}

//Unarchive will take a file system path and call 'fn' for each object that it needs for unarchiving.
//It writes to a temporary directory first and then moves this to the final location
func (a *TarArchiver) Unarchive(path string, rep Reporter, fn func(k string, w io.WriterAt) error) error {
	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()
	err = fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf)
	if err != nil {
		return errors.Wrap(err, "failed to download to temporary file")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to the beginning of file")
	}

	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat temporary file")
	}

	err = a.checkTargetDir(path)
	if err != nil {
		return err
	}

	pr := rep.StartUnarchivingProgress(tmpf.Name(), fi.Size(), tmpf)
	defer rep.StopUnarchivingProgress()

	tr := tar.NewReader(pr)
	for {
		hdr, err := tr.Next()
		switch {
		case err == io.EOF:
			return nil //EOF we're done here
		case err != nil:
			return errors.Wrap(err, "failed to read next header")
		case hdr == nil:
			continue
		}

		// the target location where the dir/file should be created
		parts := []string{path}
		parts = append(parts, strings.Split(hdr.Name, TarArchiverPathSeparator)...)
		target := filepath.Join(parts...)

		switch hdr.Typeflag {
		case tar.TypeDir: //if its a dir and it doesn't exist create it, no-op if it exists already
			err = os.MkdirAll(target, hdr.FileInfo().Mode())
			if err != nil {
				return errors.Wrap(err, "failed to create directory for entry found in tar file")
			}

		case tar.TypeReg: //regular file is written, must not exist yet
			if err = func() (err error) {
				f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, hdr.FileInfo().Mode())
				if err != nil {
					return errors.Wrap(err, "failed to open new file for tar entry ")
				}

				defer f.Close()
				if _, err := io.Copy(f, tr); err != nil {
					return errors.Wrap(err, "failed to copy archived file content")
				}

				return nil
			}(); err != nil {
				return errors.Wrap(err, "failed to extract file")
			}
		}
	}
}
<commit_msg>fix weird behavior when uploading dir w/ only 1 file<commit_after>package transferarchiver

import (
	"archive/tar"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	slashpath "path"

	"github.com/pkg/errors"
)

var (
	//TarArchiverKey configures the one object key returned by the tar Archiver
	TarArchiverKey = "archive.tar"

	//TarArchiverPathSeparator standardizes the header path for cross platform (un)archiving
	TarArchiverPathSeparator = "/"

	//ErrNoSuchDirectory is returned when an archiver expected a directory to exist
	ErrNoSuchDirectory = errors.New("directory doesn't exist")

	//ErrEmptyDirectory is returned when the archiver expected the directory to not be empty
	ErrEmptyDirectory = errors.New("directory is empty")
)

//TarArchiver will archive a directory into a single tar file
type TarArchiver struct {
	keyPrefix string
}

//NewTarArchiver will setup the tar archiver
func NewTarArchiver(opts ArchiverOptions) (a *TarArchiver, err error) {
	a = &TarArchiver{keyPrefix: opts.TarArchiverKeyPrefix}

	if a.keyPrefix != "" && !strings.HasSuffix(a.keyPrefix, "/") {
		return nil, errors.Errorf("archiver key prefix must end with a forward slash")
	}

	return a, nil
}

//tempFile will setup a temproary file that can easily be cleaned
func (a *TarArchiver) tempFile() (f *os.File, clean func(), err error) {
	f, err = ioutil.TempFile("", "tar_archiver_")
	if err != nil {
		return nil, nil, errors.Wrap(err, "failed to create temporary file")
	}

	return f, func() {
		_ = f.Close()
		_ = os.Remove(f.Name())
	}, nil
}

func (a *TarArchiver) checkTargetDir(path string) error {
	dir, err := os.Open(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return errors.Wrap(err, "failed to open directory")
		}

		err = os.Mkdir(path, 0777) //@TODO decide on permissions before umask
		if err != nil {
			return errors.Wrap(err, "failed to create directory")
		}

		dir, err = os.Open(path)
		if err != nil {
			return errors.Wrap(err, "failed open created directory")
		}
	}

	fis, err := dir.Readdirnames(1)
	if err != nil && err != io.EOF {
		return errors.Wrap(err, "failed to read directory")
	}

	if len(fis) > 0 {
		return errors.New("directory is not empty")
	}

	return nil
}

//Index calls 'fn' for all object keys that are part of the archive
func (a *TarArchiver) Index(fn func(k string) error) error {
	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey))
}

//@TODO do we want to expose this through the interface?
func (a *TarArchiver) indexFS(path string, fn func(p string, fi os.FileInfo, err error) error) error {
	if err := filepath.Walk(path, func(p string, fi os.FileInfo, err error) error {
		if fi == nil || path == p {
			return nil //this is triggered when a directory doesn't have an executable bit
		}
		if err != nil {
			return err
		}

		return fn(p, fi, nil)
	}); err != nil {
		return err
	}

	return nil
}

//Archive will archive a directory at 'path' into readable objects 'r' and calls 'fn' for each
func (a *TarArchiver) Archive(path string, rep Reporter, fn func(k string, r io.ReadSeeker, nbytes int64) error) (err error) {
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return ErrNoSuchDirectory
		}

		return err
	}

	names, err := f.Readdirnames(1)
	if len(names) == 0 {
		return ErrEmptyDirectory
	}
	if err != nil {
		return err
	}

	var totalToTar int64
	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		totalToTar += fi.Size()
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to index filesystem")
	}

	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	inc := rep.StartArchivingProgress(tmpf.Name(), totalToTar)

	defer clean()
	tw := tar.NewWriter(tmpf)
	defer tw.Close()

	if err = a.indexFS(path, func(p string, fi os.FileInfo, err error) error {
		rel, err := filepath.Rel(path, p)
		if err != nil {
			return errors.Wrap(err, "failed to determine relative path")
		}

		//write header with a filename that standardizes the Separator
		path := strings.Split(rel, string(filepath.Separator))
		hdr, err := tar.FileInfoHeader(fi, "") //@TODO find out how we handle symlinks
		if err != nil {
			return errors.Wrap(err, "failed to convert file info to tar header")
		}

		hdr.Name = strings.Join(path, TarArchiverPathSeparator)
		if err = tw.WriteHeader(hdr); err != nil {
			return errors.Wrap(err, "failed to write tar header")
		}

		if !fi.Mode().IsRegular() {
			return nil //nothing to write for dirs or symlinks
		}

		// open files for taring
		f, err := os.Open(p)
		defer f.Close()
		if err != nil {
			return errors.Wrap(err, "failed to open file for archiving")
		}

		// copy file data into tar writer
		var n int64
		if n, err = io.Copy(tw, f); err != nil {
			return errors.Wrap(err, "failed to copy file content to archive")
		}
		inc(n)
		return nil
	}); err != nil {
		return errors.Wrap(err, "failed to perform filesystem walk")
	}
	err = tw.Flush()
	if err != nil {
		return errors.Wrap(err, "failed to flush tar writer to disk")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to beginning of file")
	}

	//stop progress reporting, we're done
	rep.StopArchivingProgress()
	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat the temporary file")
	}

	return fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf, fi.Size())
}

//Unarchive will take a file system path and call 'fn' for each object that it needs for unarchiving.
//It writes to a temporary directory first and then moves this to the final location
func (a *TarArchiver) Unarchive(path string, rep Reporter, fn func(k string, w io.WriterAt) error) error {
	tmpf, clean, err := a.tempFile()
	if err != nil {
		return err
	}

	defer clean()
	err = fn(slashpath.Join(a.keyPrefix, TarArchiverKey), tmpf)
	if err != nil {
		return errors.Wrap(err, "failed to download to temporary file")
	}

	_, err = tmpf.Seek(0, 0)
	if err != nil {
		return errors.Wrap(err, "failed to seek to the beginning of file")
	}

	fi, err := tmpf.Stat()
	if err != nil {
		return errors.Wrap(err, "failed to stat temporary file")
	}

	err = a.checkTargetDir(path)
	if err != nil {
		return err
	}

	pr := rep.StartUnarchivingProgress(tmpf.Name(), fi.Size(), tmpf)
	defer rep.StopUnarchivingProgress()

	tr := tar.NewReader(pr)
	for {
		hdr, err := tr.Next()
		switch {
		case err == io.EOF:
			return nil //EOF we're done here
		case err != nil:
			return errors.Wrap(err, "failed to read next header")
		case hdr == nil:
			continue
		}

		// the target location where the dir/file should be created
		parts := []string{path}
		parts = append(parts, strings.Split(hdr.Name, TarArchiverPathSeparator)...)
		target := filepath.Join(parts...)

		switch hdr.Typeflag {
		case tar.TypeDir: //if its a dir and it doesn't exist create it, no-op if it exists already
			err = os.MkdirAll(target, hdr.FileInfo().Mode())
			if err != nil {
				return errors.Wrap(err, "failed to create directory for entry found in tar file")
			}

		case tar.TypeReg: //regular file is written, must not exist yet
			if err = func() (err error) {
				f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, hdr.FileInfo().Mode())
				if err != nil {
					return errors.Wrap(err, "failed to open new file for tar entry ")
				}

				defer f.Close()
				if _, err := io.Copy(f, tr); err != nil {
					return errors.Wrap(err, "failed to copy archived file content")
				}

				return nil
			}(); err != nil {
				return errors.Wrap(err, "failed to extract file")
			}
		}
	}
}
<|endoftext|>


<commit_before>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2017] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package node

import (
	"context"
	"errors"
	"github.com/lastbackend/lastbackend/pkg/apis/types"
	ctx "github.com/lastbackend/lastbackend/pkg/api/context"
)

type Node struct {
}

func New() *Node {
	return new(Node)
}

func (n *Node) List(c context.Context) ([]*types.Node, error) {
	var storage = ctx.Get().GetStorage()
	return storage.Node().List(c)
}

func (n *Node) Get(c context.Context, hostname string) (*types.Node, error) {
	var (
		log     = ctx.Get().GetLogger()
		storage = ctx.Get().GetStorage()
	)

	log.Debug("Node: Get node info")
	node, err := storage.Node().Get(c, hostname)
	if err != nil {
		return nil, err
	}

	return node, nil
}

func (n *Node) SetMeta(c context.Context, node *types.Node) error {
	var (
		storage = ctx.Get().GetStorage()
	)

	return storage.Node().UpdateMeta(c, &node.Meta)
}

func (n *Node) SetState(c context.Context, node *types.Node) error {
	var (
		storage = ctx.Get().GetStorage()
	)

	return storage.Node().UpdateState(c, node)
}

func (n *Node) Create(c context.Context, meta *types.NodeMeta, state *types.NodeState) (*types.Node, error) {

	var (
		storage = ctx.Get().GetStorage()
		node    = new(types.Node)
		log     = ctx.Get().GetLogger()
	)

	log.Debug("Create new Node")

	node.Meta = *meta
	node.State = *state

	if err := storage.Node().Insert(c, node); err != nil {
		return node, err
	}

	return node, nil
}

func (n *Node) PodSpecRemove(c context.Context, hostname string, spec *types.PodNodeSpec) error {

	var (
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
	)

	node, err := n.Get(c, hostname)
	if err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	log.Debug("Remove pod spec from node")
	if err := storage.Node().RemovePod(c, &node.Meta, spec); err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	// Update pod node spec

	return nil
}

func (n *Node) PodSpecUpdate(c context.Context, hostname string, spec *types.PodNodeSpec) error {
	// Get node by hostname
	// Update pod node spec
	var (
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
	)

	node, err := n.Get(c, hostname)
	if err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	log.Debug("Remove pod spec from node")
	if err := storage.Node().UpdatePod(c, &node.Meta, spec); err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	// Update pod node spec

	return nil
}

func (n *Node) Allocate(c context.Context, spec types.PodSpec) (*types.Node, error) {

	var (
		node    *types.Node
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
		memory  = int64(0)
	)

	log.Debug("Allocate Pod to Node")

	nodes, err := storage.Node().List(c)
	if err != nil {
		log.Errorf("Node: allocate: get nodes error: %s", err.Error())
		return nil, err
	}

	for _, c := range spec.Containers {
		memory += c.Quota.Memory
	}

	for _, node = range nodes {
		log.Debugf("Node: Allocate: available memory %d", node.Meta.State.Capacity)
		if node.Meta.State.Capacity.Memory > memory {
			break
		}
	}

	if node.Meta.Hostname == "" {
		log.Error("Node: Allocate: Available node not found")
		return nil, errors.New("Available node not found")
	}

	return node, nil
}
<commit_msg>fix crach when node not found<commit_after>//
// Last.Backend LLC CONFIDENTIAL
// __________________
//
// [2014] - [2017] Last.Backend LLC
// All Rights Reserved.
//
// NOTICE:  All information contained herein is, and remains
// the property of Last.Backend LLC and its suppliers,
// if any.  The intellectual and technical concepts contained
// herein are proprietary to Last.Backend LLC
// and its suppliers and may be covered by Russian Federation and Foreign Patents,
// patents in process, and are protected by trade secret or copyright law.
// Dissemination of this information or reproduction of this material
// is strictly forbidden unless prior written permission is obtained
// from Last.Backend LLC.
//

package node

import (
	"context"
	"errors"
	"github.com/lastbackend/lastbackend/pkg/apis/types"
	ctx "github.com/lastbackend/lastbackend/pkg/api/context"
)

type Node struct {
}

func New() *Node {
	return new(Node)
}

func (n *Node) List(c context.Context) ([]*types.Node, error) {
	var storage = ctx.Get().GetStorage()
	return storage.Node().List(c)
}

func (n *Node) Get(c context.Context, hostname string) (*types.Node, error) {
	var (
		log     = ctx.Get().GetLogger()
		storage = ctx.Get().GetStorage()
	)

	log.Debug("Node: Get node info")
	node, err := storage.Node().Get(c, hostname)
	if err != nil {
		return nil, err
	}

	return node, nil
}

func (n *Node) SetMeta(c context.Context, node *types.Node) error {
	var (
		storage = ctx.Get().GetStorage()
	)

	return storage.Node().UpdateMeta(c, &node.Meta)
}

func (n *Node) SetState(c context.Context, node *types.Node) error {
	var (
		storage = ctx.Get().GetStorage()
	)

	return storage.Node().UpdateState(c, node)
}

func (n *Node) Create(c context.Context, meta *types.NodeMeta, state *types.NodeState) (*types.Node, error) {

	var (
		storage = ctx.Get().GetStorage()
		node    = new(types.Node)
		log     = ctx.Get().GetLogger()
	)

	log.Debug("Create new Node")

	node.Meta = *meta
	node.State = *state

	if err := storage.Node().Insert(c, node); err != nil {
		return node, err
	}

	return node, nil
}

func (n *Node) PodSpecRemove(c context.Context, hostname string, spec *types.PodNodeSpec) error {

	var (
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
	)

	node, err := n.Get(c, hostname)
	if err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	log.Debug("Remove pod spec from node")
	if err := storage.Node().RemovePod(c, &node.Meta, spec); err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	// Update pod node spec

	return nil
}

func (n *Node) PodSpecUpdate(c context.Context, hostname string, spec *types.PodNodeSpec) error {
	// Get node by hostname
	// Update pod node spec
	var (
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
	)

	node, err := n.Get(c, hostname)
	if err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	log.Debug("Remove pod spec from node")
	if err := storage.Node().UpdatePod(c, &node.Meta, spec); err != nil {
		log.Errorf("Node: Pod spec remove: remove pod spec err: %s", err.Error())
		return err
	}

	// Update pod node spec

	return nil
}

func (n *Node) Allocate(c context.Context, spec types.PodSpec) (*types.Node, error) {

	var (
		node    *types.Node
		storage = ctx.Get().GetStorage()
		log     = ctx.Get().GetLogger()
		memory  = int64(0)
	)

	log.Debug("Allocate Pod to Node")

	nodes, err := storage.Node().List(c)
	if err != nil {
		log.Errorf("Node: allocate: get nodes error: %s", err.Error())
		return nil, err
	}

	for _, c := range spec.Containers {
		memory += c.Quota.Memory
	}

	for _, node = range nodes {
		log.Debugf("Node: Allocate: available memory %d", node.Meta.State.Capacity)
		if node.Meta.State.Capacity.Memory > memory {
			break
		}
	}

	if node == nil {
		log.Error("Node: Allocate: Available node not found")
		return nil, errors.New("Available node not found")
	}

	return node, nil
}
<|endoftext|>


<commit_before>package chunkenc

import (
	"bufio"
	"bytes"
	"io"
	"sync"

	"github.com/golang/snappy"
	"github.com/klauspost/compress/gzip"
	"github.com/pierrec/lz4"
	"github.com/prometheus/prometheus/pkg/pool"
)

// WriterPool is a pool of io.Writer
// This is used by every chunk to avoid unnecessary allocations.
type WriterPool interface {
	GetWriter(io.Writer) io.WriteCloser
	PutWriter(io.WriteCloser)
}

// ReaderPool similar to WriterPool but for reading chunks.
type ReaderPool interface {
	GetReader(io.Reader) io.Reader
	PutReader(io.Reader)
}

var (
	// Gzip is the gnu zip compression pool
	Gzip = GzipPool{level: gzip.DefaultCompression}
	// LZ4 is the l4z compression pool
	LZ4_64k  = LZ4Pool{bufferSize: 1 << 16}
	LZ4_256k = LZ4Pool{bufferSize: 1 << 18}
	LZ4_1M   = LZ4Pool{bufferSize: 1 << 20}
	LZ4_4M   = LZ4Pool{bufferSize: 1 << 22}

	// Snappy is the snappy compression pool
	Snappy SnappyPool
	// Noop is the no compression pool
	Noop NoopPool

	// BufReaderPool is bufio.Reader pool
	BufReaderPool = &BufioReaderPool{
		pool: sync.Pool{
			New: func() interface{} { return bufio.NewReader(nil) },
		},
	}
	// BytesBufferPool is a bytes buffer used for lines decompressed.
	// Buckets [0.5KB,1KB,2KB,4KB,8KB]
	BytesBufferPool          = pool.New(1<<9, 1<<13, 2, func(size int) interface{} { return make([]byte, 0, size) })
	serializeBytesBufferPool = sync.Pool{
		New: func() interface{} {
			return &bytes.Buffer{}
		},
	}
)

func getWriterPool(enc Encoding) WriterPool {
	return getReaderPool(enc).(WriterPool)
}

func getReaderPool(enc Encoding) ReaderPool {
	switch enc {
	case EncGZIP:
		return &Gzip
	case EncLZ4_64k:
		return &LZ4_64k
	case EncLZ4_256k:
		return &LZ4_256k
	case EncLZ4_1M:
		return &LZ4_1M
	case EncLZ4_4M:
		return &LZ4_4M
	case EncSnappy:
		return &Snappy
	case EncNone:
		return &Noop
	default:
		panic("unknown encoding")
	}
}

// GzipPool is a gun zip compression pool
type GzipPool struct {
	readers sync.Pool
	writers sync.Pool
	level   int
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *GzipPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*gzip.Reader)
		err := reader.Reset(src)
		if err != nil {
			panic(err)
		}
		return reader
	}
	reader, err := gzip.NewReader(src)
	if err != nil {
		panic(err)
	}
	return reader
}

// PutReader places back in the pool a CompressionReader
func (pool *GzipPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *GzipPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*gzip.Writer)
		writer.Reset(dst)
		return writer
	}

	level := pool.level
	if level == 0 {
		level = gzip.DefaultCompression
	}
	w, err := gzip.NewWriterLevel(dst, level)
	if err != nil {
		panic(err) // never happens, error is only returned on wrong compression level.
	}
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *GzipPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type LZ4Pool struct {
	readers    sync.Pool
	writers    sync.Pool
	bufferSize int // available values: 1<<16 (64k), 1<<18 (256k), 1<<20 (1M), 1<<22 (4M). Defaults to 4MB, if not set.
}

// lz4Reader is simple wrapper around *lz4.Reader, which remembers max used block size,
// as reported by this reader. It is used to determine whether we want to reuse it,
// or throw away and garbage-collect.
type lz4Reader struct {
	r            *lz4.Reader
	maxBlockSize int
}

func (l *lz4Reader) Read(p []byte) (n int, err error) {
	return l.r.Read(p)
}

func (l *lz4Reader) Reset(src io.Reader) {
	l.r.Reset(src)
}

func (l *lz4Reader) onBlockDone(_ int) {
	// remember max block size used.
	if l.r.BlockMaxSize > l.maxBlockSize {
		l.maxBlockSize = l.r.BlockMaxSize
	}
}

func newLz4Reader(src io.Reader) *lz4Reader {
	lz4r := lz4.NewReader(src)
	r := &lz4Reader{r: lz4r}
	lz4r.OnBlockDone = r.onBlockDone
	return r
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *LZ4Pool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*lz4Reader)
		reader.Reset(src)
		return reader
	}
	// no need to set buffer size here. Reader uses buffer size based on
	// LZ4 header that it is reading.
	r := newLz4Reader(src)
	return r
}

// PutReader places back in the pool a CompressionReader
func (pool *LZ4Pool) PutReader(reader io.Reader) {
	r := reader.(*lz4Reader)
	if r.maxBlockSize > pool.bufferSize {
		// Readers base their buffer size based on headers from LZ4 stream.
		// If this reader uses bigger buffer than what we use currently, don't pool it.
		// Reading from a couple of chunks that used big buffer sizes could otherwise quickly lead
		// to high pooled memory usage.
		return
	}
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *LZ4Pool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*lz4.Writer)
		writer.Reset(dst)
		return writer
	}
	w := lz4.NewWriter(dst)
	w.BlockMaxSize = pool.bufferSize
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *LZ4Pool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type SnappyPool struct {
	readers sync.Pool
	writers sync.Pool
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *SnappyPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*snappy.Reader)
		reader.Reset(src)
		return reader
	}
	return snappy.NewReader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *SnappyPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *SnappyPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*snappy.Writer)
		writer.Reset(dst)
		return writer
	}
	return snappy.NewBufferedWriter(dst)
}

// PutWriter places back in the pool a CompressionWriter
func (pool *SnappyPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type NoopPool struct{}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *NoopPool) GetReader(src io.Reader) io.Reader {
	return src
}

// PutReader places back in the pool a CompressionReader
func (pool *NoopPool) PutReader(reader io.Reader) {}

type noopCloser struct {
	io.Writer
}

func (noopCloser) Close() error { return nil }

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *NoopPool) GetWriter(dst io.Writer) io.WriteCloser {
	return noopCloser{dst}
}

// PutWriter places back in the pool a CompressionWriter
func (pool *NoopPool) PutWriter(writer io.WriteCloser) {}

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool sync.Pool
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	bufPool.pool.Put(b)
}
<commit_msg>Undo small change.<commit_after>package chunkenc

import (
	"bufio"
	"bytes"
	"io"
	"sync"

	"github.com/golang/snappy"
	"github.com/klauspost/compress/gzip"
	"github.com/pierrec/lz4"
	"github.com/prometheus/prometheus/pkg/pool"
)

// WriterPool is a pool of io.Writer
// This is used by every chunk to avoid unnecessary allocations.
type WriterPool interface {
	GetWriter(io.Writer) io.WriteCloser
	PutWriter(io.WriteCloser)
}

// ReaderPool similar to WriterPool but for reading chunks.
type ReaderPool interface {
	GetReader(io.Reader) io.Reader
	PutReader(io.Reader)
}

var (
	// Gzip is the gnu zip compression pool
	Gzip = GzipPool{level: gzip.DefaultCompression}
	// LZ4 is the l4z compression pool
	LZ4_64k  = LZ4Pool{bufferSize: 1 << 16}
	LZ4_256k = LZ4Pool{bufferSize: 1 << 18}
	LZ4_1M   = LZ4Pool{bufferSize: 1 << 20}
	LZ4_4M   = LZ4Pool{bufferSize: 1 << 22}

	// Snappy is the snappy compression pool
	Snappy SnappyPool
	// Noop is the no compression pool
	Noop NoopPool

	// BufReaderPool is bufio.Reader pool
	BufReaderPool = &BufioReaderPool{
		pool: sync.Pool{
			New: func() interface{} { return bufio.NewReader(nil) },
		},
	}
	// BytesBufferPool is a bytes buffer used for lines decompressed.
	// Buckets [0.5KB,1KB,2KB,4KB,8KB]
	BytesBufferPool          = pool.New(1<<9, 1<<13, 2, func(size int) interface{} { return make([]byte, 0, size) })
	serializeBytesBufferPool = sync.Pool{
		New: func() interface{} {
			return &bytes.Buffer{}
		},
	}
)

func getWriterPool(enc Encoding) WriterPool {
	return getReaderPool(enc).(WriterPool)
}

func getReaderPool(enc Encoding) ReaderPool {
	switch enc {
	case EncGZIP:
		return &Gzip
	case EncLZ4_64k:
		return &LZ4_64k
	case EncLZ4_256k:
		return &LZ4_256k
	case EncLZ4_1M:
		return &LZ4_1M
	case EncLZ4_4M:
		return &LZ4_4M
	case EncSnappy:
		return &Snappy
	case EncNone:
		return &Noop
	default:
		panic("unknown encoding")
	}
}

// GzipPool is a gun zip compression pool
type GzipPool struct {
	readers sync.Pool
	writers sync.Pool
	level   int
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *GzipPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*gzip.Reader)
		err := reader.Reset(src)
		if err != nil {
			panic(err)
		}
		return reader
	}
	reader, err := gzip.NewReader(src)
	if err != nil {
		panic(err)
	}
	return reader
}

// PutReader places back in the pool a CompressionReader
func (pool *GzipPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *GzipPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*gzip.Writer)
		writer.Reset(dst)
		return writer
	}

	level := pool.level
	if level == 0 {
		level = gzip.DefaultCompression
	}
	w, err := gzip.NewWriterLevel(dst, level)
	if err != nil {
		panic(err) // never happens, error is only returned on wrong compression level.
	}
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *GzipPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type LZ4Pool struct {
	readers    sync.Pool
	writers    sync.Pool
	bufferSize int // available values: 1<<16 (64k), 1<<18 (256k), 1<<20 (1M), 1<<22 (4M). Defaults to 4MB, if not set.
}

// lz4Reader is simple wrapper around *lz4.Reader, which remembers max used block size,
// as reported by this reader. It is used to determine whether we want to reuse it,
// or throw away and garbage-collect.
type lz4Reader struct {
	r            *lz4.Reader
	maxBlockSize int
}

func (l *lz4Reader) Read(p []byte) (n int, err error) {
	return l.r.Read(p)
}

func (l *lz4Reader) Reset(src io.Reader) {
	l.r.Reset(src)
}

func (l *lz4Reader) onBlockDone(_ int) {
	// remember max block size used.
	if l.r.BlockMaxSize > l.maxBlockSize {
		l.maxBlockSize = l.r.BlockMaxSize
	}
}

func newLz4Reader(src io.Reader) *lz4Reader {
	lz4r := lz4.NewReader(src)
	r := &lz4Reader{r: lz4r}
	lz4r.OnBlockDone = r.onBlockDone
	return r
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *LZ4Pool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*lz4Reader)
		reader.Reset(src)
		return reader
	}
	// no need to set buffer size here. Reader uses buffer size based on
	// LZ4 header that it is reading.
	return newLz4Reader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *LZ4Pool) PutReader(reader io.Reader) {
	r := reader.(*lz4Reader)
	if r.maxBlockSize > pool.bufferSize {
		// Readers base their buffer size based on headers from LZ4 stream.
		// If this reader uses bigger buffer than what we use currently, don't pool it.
		// Reading from a couple of chunks that used big buffer sizes could otherwise quickly lead
		// to high pooled memory usage.
		return
	}
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *LZ4Pool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*lz4.Writer)
		writer.Reset(dst)
		return writer
	}
	w := lz4.NewWriter(dst)
	w.BlockMaxSize = pool.bufferSize
	return w
}

// PutWriter places back in the pool a CompressionWriter
func (pool *LZ4Pool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type SnappyPool struct {
	readers sync.Pool
	writers sync.Pool
}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *SnappyPool) GetReader(src io.Reader) io.Reader {
	if r := pool.readers.Get(); r != nil {
		reader := r.(*snappy.Reader)
		reader.Reset(src)
		return reader
	}
	return snappy.NewReader(src)
}

// PutReader places back in the pool a CompressionReader
func (pool *SnappyPool) PutReader(reader io.Reader) {
	pool.readers.Put(reader)
}

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *SnappyPool) GetWriter(dst io.Writer) io.WriteCloser {
	if w := pool.writers.Get(); w != nil {
		writer := w.(*snappy.Writer)
		writer.Reset(dst)
		return writer
	}
	return snappy.NewBufferedWriter(dst)
}

// PutWriter places back in the pool a CompressionWriter
func (pool *SnappyPool) PutWriter(writer io.WriteCloser) {
	pool.writers.Put(writer)
}

type NoopPool struct{}

// GetReader gets or creates a new CompressionReader and reset it to read from src
func (pool *NoopPool) GetReader(src io.Reader) io.Reader {
	return src
}

// PutReader places back in the pool a CompressionReader
func (pool *NoopPool) PutReader(reader io.Reader) {}

type noopCloser struct {
	io.Writer
}

func (noopCloser) Close() error { return nil }

// GetWriter gets or creates a new CompressionWriter and reset it to write to dst
func (pool *NoopPool) GetWriter(dst io.Writer) io.WriteCloser {
	return noopCloser{dst}
}

// PutWriter places back in the pool a CompressionWriter
func (pool *NoopPool) PutWriter(writer io.WriteCloser) {}

// BufioReaderPool is a bufio reader that uses sync.Pool.
type BufioReaderPool struct {
	pool sync.Pool
}

// Get returns a bufio.Reader which reads from r. The buffer size is that of the pool.
func (bufPool *BufioReaderPool) Get(r io.Reader) *bufio.Reader {
	buf := bufPool.pool.Get().(*bufio.Reader)
	buf.Reset(r)
	return buf
}

// Put puts the bufio.Reader back into the pool.
func (bufPool *BufioReaderPool) Put(b *bufio.Reader) {
	bufPool.pool.Put(b)
}
<|endoftext|>