<commit_before>//
// store.go
//

package srnd

import (

  "github.com/gographics/imagick/imagick"

  "bytes"
  "crypto/sha512"
  "encoding/base32"
  "encoding/base64"
  "fmt"
  "io"
  //"io/ioutil"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "net/textproto"
  "os"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter
  MessageVerifier
  AttachmentReader

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders

  // get our temp directory for articles
  TempDir() string
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
}

func (self articleStore) GenerateThumbnail(infname string) (err error) {
  wand := imagick.NewMagickWand()
  defer wand.Destroy()
  // read image source
  err = wand.ReadImage(self.AttachmentFilepath(infname))
  if err == nil {
    // get size
    h := wand.GetImageHeight()
    w := wand.GetImageWidth()

    // calculate scale parameters
    var scale, th, tw float64
    scale = 180
    modifier := scale / float64(w)
    th = modifier * float64(h)
    tw = modifier * float64(w)
    // scale it
    err = wand.ScaleImage(uint(tw), uint(th))
    if err == nil {
      err = wand.WriteImage(self.ThumbnailFilepath(infname))
    }
  }
  return
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {
  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }
  for _, att := range nntp.Attachments() {
    fpath := att.Filepath()

    upload := self.AttachmentFilepath(fpath)

    // save attachment
    log.Println("save attachment", att.Filename(), "to", upload)
    f, err = os.Create(upload)
    if err == nil {
      err = att.WriteTo(f)
      f.Close()
    }
    if err != nil {
      return
    }

    // generate thumbanils
    if att.NeedsThumbnail() {
      err = self.GenerateThumbnail(fpath)
      if err != nil {
        log.Println("failed to generate thumbnail", err)
        return
      }
    }
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  nntp := nntpArticle{}

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := msg.Header.Get("Content-Type")
    _, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          log.Println("part has content type", part_type)
          // parse content type
          media_type, _, err := mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              encoding := hdr.Get("Content-Transfer-Encoding")
              if encoding == "base64" {
                att := self.ReadAttachmentFromMimePart(part, true)
                if att != nil {
                  att.WriteTo(&nntp.message.body)
                }
              } else {
                nntp.message.body.ReadFrom(part)
              }
              if nntp.message.header == nil {
                nntp.message.header = make(textproto.MIMEHeader)
              }
              nntp.message.header.Set("Content-Type", part_type)
            } else {
              // non plaintext gets added to attachments
              att := self.ReadAttachmentFromMimePart(part, true)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nntp, err
        }
      }
    } else {
      _, err = nntp.message.body.ReadFrom(msg.Body)
    }
  } else {
    log.Println("failed to read message", err)
  }
  return nntp, err
}

func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  // write headers
  for hdr, hdr_vals := range(nntp.Headers()) {
    for _ , hdr_val := range hdr_vals {
      _, err = io.WriteString(wr, fmt.Sprintf("%s: %s\n", hdr, hdr_val))
      if err != nil {
        return
      }
    }
  }
  // done headers
  _, err = io.WriteString(wr, "\n")
  if err != nil {
    return
  }
  // write body
  err = nntp.WriteBody(wr)
  return
}

func (self articleStore) VerifyMessage(nntp NNTPMessage) (NNTPMessage, error) {
  // TODO: implement
  return nntp, nil
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  return filepath.Join(self.thumbs, fname)
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return self.database.HasArticle(messageID)
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func (self articleStore) ReadAttachmentFromMimePart(part *multipart.Part, decode bool) NNTPAttachment {
  var buff bytes.Buffer
  content_type := part.Header.Get("Content-Type")
  media_type, _ , err := mime.ParseMediaType(content_type)

  fname := part.FileName()
  idx := strings.LastIndex(fname, ".")
  ext := ".txt"
  if idx > 0 {
    ext = fname[idx:]
  }

  if decode {
    reader := base64.NewDecoder(base64.StdEncoding, part)
    _, err = io.Copy(&buff, reader)
  } else {
    _, err = io.Copy(&buff, part)
  }
  if err != nil {
    log.Println("failed to read attachment from mimepart", err)
    return nil
  }
  sha := sha512.Sum512(buff.Bytes())
  hashstr := base32.StdEncoding.EncodeToString(sha[:])
  fpath := hashstr+ext
  hdr := part.Header
  return nntpAttachment{
    header: hdr,
    body: buff,
    mime: media_type,
    filename: fname,
    filepath: fpath,
    ext: ext,
    hash: sha[:],
  }
}
<commit_msg>detect transfer encoding in part parser<commit_after>//
// store.go
//

package srnd

import (

  "github.com/gographics/imagick/imagick"

  "bytes"
  "crypto/sha512"
  "encoding/base32"
  "encoding/base64"
  "fmt"
  "io"
  //"io/ioutil"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "net/textproto"
  "os"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter
  MessageVerifier
  AttachmentReader

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders

  // get our temp directory for articles
  TempDir() string
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
}

func (self articleStore) GenerateThumbnail(infname string) (err error) {
  wand := imagick.NewMagickWand()
  defer wand.Destroy()
  // read image source
  err = wand.ReadImage(self.AttachmentFilepath(infname))
  if err == nil {
    // get size
    h := wand.GetImageHeight()
    w := wand.GetImageWidth()

    // calculate scale parameters
    var scale, th, tw float64
    scale = 180
    modifier := scale / float64(w)
    th = modifier * float64(h)
    tw = modifier * float64(w)
    // scale it
    err = wand.ScaleImage(uint(tw), uint(th))
    if err == nil {
      err = wand.WriteImage(self.ThumbnailFilepath(infname))
    }
  }
  return
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {
  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }
  for _, att := range nntp.Attachments() {
    fpath := att.Filepath()

    upload := self.AttachmentFilepath(fpath)

    // save attachment
    log.Println("save attachment", att.Filename(), "to", upload)
    f, err = os.Create(upload)
    if err == nil {
      err = att.WriteTo(f)
      f.Close()
    }
    if err != nil {
      return
    }

    // generate thumbanils
    if att.NeedsThumbnail() {
      err = self.GenerateThumbnail(fpath)
      if err != nil {
        log.Println("failed to generate thumbnail", err)
        return
      }
    }
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  nntp := nntpArticle{}

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := msg.Header.Get("Content-Type")
    _, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          log.Println("part has content type", part_type)
          // parse content type
          media_type, _, err := mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              att := self.ReadAttachmentFromMimePart(part, true)
              if att != nil {
                att.WriteTo(&nntp.message.body)
              }
              if nntp.message.header == nil {
                nntp.message.header = make(textproto.MIMEHeader)
              }
              nntp.message.header.Set("Content-Type", part_type)
            } else {
              // non plaintext gets added to attachments
              att := self.ReadAttachmentFromMimePart(part, true)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nntp, err
        }
      }
    } else {
      _, err = nntp.message.body.ReadFrom(msg.Body)
    }
  } else {
    log.Println("failed to read message", err)
  }
  return nntp, err
}

func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  // write headers
  for hdr, hdr_vals := range(nntp.Headers()) {
    for _ , hdr_val := range hdr_vals {
      _, err = io.WriteString(wr, fmt.Sprintf("%s: %s\n", hdr, hdr_val))
      if err != nil {
        return
      }
    }
  }
  // done headers
  _, err = io.WriteString(wr, "\n")
  if err != nil {
    return
  }
  // write body
  err = nntp.WriteBody(wr)
  return
}

func (self articleStore) VerifyMessage(nntp NNTPMessage) (NNTPMessage, error) {
  // TODO: implement
  return nntp, nil
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  return filepath.Join(self.thumbs, fname)
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return self.database.HasArticle(messageID)
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func (self articleStore) ReadAttachmentFromMimePart(part *multipart.Part, decode bool) NNTPAttachment {
  var buff bytes.Buffer
  content_type := part.Header.Get("Content-Type")
  media_type, _ , err := mime.ParseMediaType(content_type)

  fname := part.FileName()
  idx := strings.LastIndex(fname, ".")
  ext := ".txt"
  if idx > 0 {
    ext = fname[idx:]
  }

  transfer_encoding := part.Header.Get("Content-Transfer-Encoding")

  if transfer_encoding == "base64" {
    reader := base64.NewDecoder(base64.StdEncoding, part)
    _, err = io.Copy(&buff, reader)
  } else {
    _, err = io.Copy(&buff, part)
  }
  if err != nil {
    log.Println("failed to read attachment from mimepart", err)
    return nil
  }
  sha := sha512.Sum512(buff.Bytes())
  hashstr := base32.StdEncoding.EncodeToString(sha[:])
  fpath := hashstr+ext
  hdr := part.Header
  return nntpAttachment{
    header: hdr,
    body: buff,
    mime: media_type,
    filename: fname,
    filepath: fpath,
    ext: ext,
    hash: sha[:],
  }
}
<|endoftext|>


<commit_before>package irc

import . "launchpad.net/gocheck"

func (s *s) TestChannelFinder(c *C) {
	finder, err := CreateChannelFinder(`*+?[]()-^`)
	c.Assert(err, IsNil)
	c.Assert(finder.channelRegexp, NotNil)
	c.Assert(len(finder.FindChannels(")channel")), Equals, 1)
	c.Assert(finder.IsChannel(")channel"), Equals, true)
}
<commit_msg>Adds an error handling test for channel finder.<commit_after>package irc

import . "launchpad.net/gocheck"

func (s *s) TestChannelFinder(c *C) {
	finder, err := CreateChannelFinder(`~&*+?[]()-^`)
	c.Assert(err, IsNil)
	c.Assert(finder.channelRegexp, NotNil)
	c.Assert(len(finder.FindChannels(")channel")), Equals, 1)
	c.Assert(finder.IsChannel(")channel"), Equals, true)
}

func (s *s) TestChannelFinder_Error(c *C) {
	_, err := CreateChannelFinder(`H`)
	c.Assert(err, NotNil)
}
<|endoftext|>


<commit_before>package logic

import (
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/jclebreton/opensirene/database"
	"github.com/jclebreton/opensirene/opendata/gouvfr/sirene"
)

// ResetDatabase truncates all database tables
func ResetDatabase(pgxClient *database.PgxClient) error {
	if _, err := pgxClient.Conn.Exec("TRUNCATE table history, enterprises, temp_incremental"); err != nil {
		return err
	}
	return nil
}

// ImportRemoteFiles is the way to remote files to database
func ImportRemoteFiles(pgxClient *database.PgxClient, remoteFiles sirene.RemoteFiles, dPath string) error {
	var err error

	//Lock database for import
	dbMutex := database.NewPgxMutex(pgxClient)
	if err = dbMutex.Lock(); err != nil {
		return err
	}
	defer func() {
		if err = dbMutex.Unlock(); err != nil {
			logrus.WithError(err).Warning("Couldn't freeze database mutex")
		}
	}()

	//Download an extract
	if err = sirene.Do(remoteFiles, 4, dPath); err != nil {
		return errors.Wrap(err, "Couldn't retrieve files")
	}

	// Convert them
	cis, err := toCSVImport(remoteFiles)
	if err != nil {
		return errors.Wrap(err, "Couldn't convert to CSVImport")
	}

	//Import
	tracker := NewTracker(pgxClient)
	if err = cis.importCSVFiles(pgxClient, tracker); err != nil {
		return errors.Wrap(err, "Import error")
	}

	return nil
}

// ToCSVImport converts a slice of RemoteFile to a slice of CSVImport.
// It expects that at least one file was extracted
func toCSVImport(rfs sirene.RemoteFiles) (CSVImports, error) {
	var out CSVImports
	for _, rf := range rfs {
		out = append(out, &sirene.CSVImport{
			Path:    rf.ExtractedFiles[0],
			Kind:    rf.Type,
			ZipName: rf.FileName,
		})
	}
	return out, nil
}

// CSVImports is a slice of pointer to CSVImport
type CSVImports []*sirene.CSVImport

// Import will import each CSVImport present in the slice
func (c CSVImports) importCSVFiles(pgxClient *database.PgxClient, tracker Tracker) error {
	var err error
	for _, ci := range c {

		if ci.Kind == sirene.StockType {
			if err = tracker.Truncate(); err != nil {
				return errors.Wrap(err, "Unable to reset history before import stock file")
			}
		}

		if err = ci.Copy(pgxClient.Conn); err != nil {
			if e := tracker.Save(ci.ZipName, false, err.Error()); e != nil {
				return errors.Wrap(err, e.Error())
			}
			return errors.Wrap(err, "Couldn't copy")
		}

		if err = ci.Update(pgxClient.Conn); err != nil {
			if e := tracker.Save(ci.ZipName, false, err.Error()); e != nil {
				return errors.Wrap(err, e.Error())
			}
			return errors.Wrap(err, "Couldn't apply update")
		}

		// Save step to db
		err = tracker.Save(ci.ZipName, true, "")
		if err != nil {
			return errors.Wrap(err, "Couldn't log")
		}
	}
	return nil
}
<commit_msg>Fix comment<commit_after>package logic

import (
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/jclebreton/opensirene/database"
	"github.com/jclebreton/opensirene/opendata/gouvfr/sirene"
)

// ResetDatabase truncates all database tables
func ResetDatabase(pgxClient *database.PgxClient) error {
	if _, err := pgxClient.Conn.Exec("TRUNCATE table history, enterprises, temp_incremental"); err != nil {
		return err
	}
	return nil
}

// ImportRemoteFiles is the way to remote files to database
func ImportRemoteFiles(pgxClient *database.PgxClient, remoteFiles sirene.RemoteFiles, dPath string) error {
	var err error

	//Lock database during import
	dbMutex := database.NewPgxMutex(pgxClient)
	if err = dbMutex.Lock(); err != nil {
		return err
	}
	defer func() {
		if err = dbMutex.Unlock(); err != nil {
			logrus.WithError(err).Warning("Couldn't freeze database mutex")
		}
	}()

	//Download an extract
	if err = sirene.Do(remoteFiles, 4, dPath); err != nil {
		return errors.Wrap(err, "Couldn't retrieve files")
	}

	// Convert them
	cis, err := toCSVImport(remoteFiles)
	if err != nil {
		return errors.Wrap(err, "Couldn't convert to CSVImport")
	}

	//Import
	tracker := NewTracker(pgxClient)
	if err = cis.importCSVFiles(pgxClient, tracker); err != nil {
		return errors.Wrap(err, "Import error")
	}

	return nil
}

// ToCSVImport converts a slice of RemoteFile to a slice of CSVImport.
// It expects that at least one file was extracted
func toCSVImport(rfs sirene.RemoteFiles) (CSVImports, error) {
	var out CSVImports
	for _, rf := range rfs {
		out = append(out, &sirene.CSVImport{
			Path:    rf.ExtractedFiles[0],
			Kind:    rf.Type,
			ZipName: rf.FileName,
		})
	}
	return out, nil
}

// CSVImports is a slice of pointer to CSVImport
type CSVImports []*sirene.CSVImport

// Import will import each CSVImport present in the slice
func (c CSVImports) importCSVFiles(pgxClient *database.PgxClient, tracker Tracker) error {
	var err error
	for _, ci := range c {

		if ci.Kind == sirene.StockType {
			if err = tracker.Truncate(); err != nil {
				return errors.Wrap(err, "Unable to reset history before import stock file")
			}
		}

		if err = ci.Copy(pgxClient.Conn); err != nil {
			if e := tracker.Save(ci.ZipName, false, err.Error()); e != nil {
				return errors.Wrap(err, e.Error())
			}
			return errors.Wrap(err, "Couldn't copy")
		}

		if err = ci.Update(pgxClient.Conn); err != nil {
			if e := tracker.Save(ci.ZipName, false, err.Error()); e != nil {
				return errors.Wrap(err, e.Error())
			}
			return errors.Wrap(err, "Couldn't apply update")
		}

		// Save step to db
		err = tracker.Save(ci.ZipName, true, "")
		if err != nil {
			return errors.Wrap(err, "Couldn't log")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package plugin

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
)

//HTTP Constants
const (
	MaxTries            = 5
	RequestTimeout      = time.Minute
	AuthHeader          = "X_AUTH_HEADER"
	RequestedWithValue  = "X-CDS-SDK"
	RequestedWithHeader = "X-Requested-With"
)

var (
	auth   IOptions
	client *http.Client
	//Trace is a debug logger
	Trace *log.Logger
)

//Common is the base plugin struct every plugin should be composed by
type Common struct{}

//SetTrace is for debug
func SetTrace(traceHandle io.Writer) {
	Trace = log.New(traceHandle, "TRACE: ", log.Ldate|log.Ltime)
}

//Init is a common function for all plugins
func (p *Common) Init(o IOptions) string {
	SetTrace(ioutil.Discard)
	auth = o

	if auth.TLSSkipVerify() {
		client = &http.Client{
			Transport: &httpcontrol.Transport{
				RequestTimeout: RequestTimeout,
				MaxTries:       MaxTries,
				TLSClientConfig: &tls.Config{
					InsecureSkipVerify: true,
				},
			},
		}
	} else {
		client = &http.Client{
			Transport: &httpcontrol.Transport{
				RequestTimeout: RequestTimeout,
				MaxTries:       MaxTries,
			},
		}
	}
	return "plugin: initialized on " + o.GetURL()
}

// request executes an authentificated HTTP request on $path given $method
func request(method string, path string, args []byte) ([]byte, int, error) {
	if auth == nil {
		return []byte{}, 0, errors.New("Auth must be initialized")
	}
	var req *http.Request
	var err error
	if args != nil {
		req, err = http.NewRequest(method, auth.GetURL()+path, bytes.NewReader(args))
		if err != nil {
			return nil, 0, err
		}
	} else {
		req, err = http.NewRequest(method, auth.GetURL()+path, nil)
		if err != nil {
			return nil, 0, err
		}
	}

	basedHash := base64.StdEncoding.EncodeToString([]byte(auth.Hash()))
	req.Header.Set(AuthHeader, basedHash)
	req.Header.Set(RequestedWithHeader, RequestedWithValue)
	resp, err := client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	code := resp.StatusCode

	var body []byte
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, code, err
	}

	return body, code, nil
}

// Log struct holds a single line of build log
type Log struct {
	ID                 int64                `json:"id"`
	PipelineBuildJobID int64                `json:"pipeline_build_job_id"`
	PipelineBuildID    int64                `json:"pipeline_build_id"`
	Start              *timestamp.Timestamp `json:"start"`
	LastModified       *timestamp.Timestamp `json:"last_modified"`
	Done               *timestamp.Timestamp `json:"done"`
	StepOrder          int                  `json:"step_order"`
	Value              string               `json:"value"`
}

//SendLog send logs to CDS engine for the current
func SendLog(j IJob, format string, i ...interface{}) error {
	if j == nil {
		//If action is nil: do nothing
		return nil
	}
	Trace.Printf(format+"\n", i)

	now, _ := ptypes.TimestampProto(time.Now())

	s := fmt.Sprintf(format, i...)
	l := Log{
		PipelineBuildJobID: j.ID(),
		PipelineBuildID:    j.PipelineBuildID(),
		Start:              now,
		StepOrder:          j.StepOrder(),
		Value:              s,
		LastModified:       now,
		Done:               &timestamp.Timestamp{},
	}

	data, err := json.Marshal(l)
	if err != nil {
		return err
	}

	path := fmt.Sprintf("/build/%d/log", j.ID())
	_, _, err = request("POST", path, data)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fix (plugins): log format (#425)<commit_after>package plugin

import (
	"bytes"
	"crypto/tls"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/facebookgo/httpcontrol"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/timestamp"
)

//HTTP Constants
const (
	MaxTries            = 5
	RequestTimeout      = time.Minute
	AuthHeader          = "X_AUTH_HEADER"
	RequestedWithValue  = "X-CDS-SDK"
	RequestedWithHeader = "X-Requested-With"
)

var (
	auth   IOptions
	client *http.Client
	//Trace is a debug logger
	Trace *log.Logger
)

//Common is the base plugin struct every plugin should be composed by
type Common struct{}

//SetTrace is for debug
func SetTrace(traceHandle io.Writer) {
	Trace = log.New(traceHandle, "TRACE: ", log.Ldate|log.Ltime)
}

//Init is a common function for all plugins
func (p *Common) Init(o IOptions) string {
	SetTrace(ioutil.Discard)
	auth = o

	if auth.TLSSkipVerify() {
		client = &http.Client{
			Transport: &httpcontrol.Transport{
				RequestTimeout: RequestTimeout,
				MaxTries:       MaxTries,
				TLSClientConfig: &tls.Config{
					InsecureSkipVerify: true,
				},
			},
		}
	} else {
		client = &http.Client{
			Transport: &httpcontrol.Transport{
				RequestTimeout: RequestTimeout,
				MaxTries:       MaxTries,
			},
		}
	}
	return "plugin: initialized on " + o.GetURL()
}

// request executes an authentificated HTTP request on $path given $method
func request(method string, path string, args []byte) ([]byte, int, error) {
	if auth == nil {
		return []byte{}, 0, errors.New("Auth must be initialized")
	}
	var req *http.Request
	var err error
	if args != nil {
		req, err = http.NewRequest(method, auth.GetURL()+path, bytes.NewReader(args))
		if err != nil {
			return nil, 0, err
		}
	} else {
		req, err = http.NewRequest(method, auth.GetURL()+path, nil)
		if err != nil {
			return nil, 0, err
		}
	}

	basedHash := base64.StdEncoding.EncodeToString([]byte(auth.Hash()))
	req.Header.Set(AuthHeader, basedHash)
	req.Header.Set(RequestedWithHeader, RequestedWithValue)
	resp, err := client.Do(req)
	if err != nil {
		return nil, 0, err
	}
	defer resp.Body.Close()

	code := resp.StatusCode

	var body []byte
	body, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, code, err
	}

	return body, code, nil
}

// Log struct holds a single line of build log
type Log struct {
	ID                 int64                `json:"id"`
	PipelineBuildJobID int64                `json:"pipelineBuildJobID"`
	PipelineBuildID    int64                `json:"pipelineBuildID"`
	Start              *timestamp.Timestamp `json:"start"`
	LastModified       *timestamp.Timestamp `json:"lastModified"`
	Done               *timestamp.Timestamp `json:"done"`
	StepOrder          int                  `json:"stepOrder"`
	Value              string               `json:"val"`
}

//SendLog send logs to CDS engine for the current
func SendLog(j IJob, format string, i ...interface{}) error {
	if j == nil {
		//If action is nil: do nothing
		return nil
	}
	Trace.Printf(format+"\n", i)

	now, _ := ptypes.TimestampProto(time.Now())

	s := fmt.Sprintf(format, i...)
	l := Log{
		PipelineBuildJobID: j.ID(),
		PipelineBuildID:    j.PipelineBuildID(),
		Start:              now,
		StepOrder:          j.StepOrder(),
		Value:              s,
		LastModified:       now,
		Done:               &timestamp.Timestamp{},
	}

	data, err := json.Marshal(l)
	if err != nil {
		return err
	}

	path := fmt.Sprintf("/build/%d/log", j.ID())
	_, _, err = request("POST", path, data)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"sort"
	"sync"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/config/module"
)

// InputMode defines what sort of input will be asked for when Input
// is called on Context.
type InputMode byte

const (
	// InputModeVar asks for variables
	InputModeVar InputMode = 1 << iota

	// InputModeProvider asks for provider variables
	InputModeProvider

	// InputModeStd is the standard operating mode and asks for both variables
	// and providers.
	InputModeStd = InputModeVar | InputModeProvider
)

// ContextOpts are the user-configurable options to create a context with
// NewContext.
type ContextOpts struct {
	Diff         *Diff
	Hooks        []Hook
	Module       *module.Tree
	Parallelism  int
	State        *State
	Providers    map[string]ResourceProviderFactory
	Provisioners map[string]ResourceProvisionerFactory
	Variables    map[string]string

	UIInput UIInput
}

// Context represents all the context that Terraform needs in order to
// perform operations on infrastructure. This structure is built using
// NewContext. See the documentation for that.
type Context struct {
	diff         *Diff
	diffLock     sync.RWMutex
	hooks        []Hook
	module       *module.Tree
	providers    map[string]ResourceProviderFactory
	provisioners map[string]ResourceProvisionerFactory
	sh           *stopHook
	state        *State
	stateLock    sync.RWMutex
	uiInput      UIInput
	variables    map[string]string

	l                   sync.Mutex // Lock acquired during any task
	providerInputConfig map[string]map[string]interface{}
	runCh               <-chan struct{}
}

// NewContext creates a new Context structure.
//
// Once a Context is creator, the pointer values within ContextOpts
// should not be mutated in any way, since the pointers are copied, not
// the values themselves.
func NewContext(opts *ContextOpts) *Context {
	// Copy all the hooks and add our stop hook. We don't append directly
	// to the Config so that we're not modifying that in-place.
	sh := new(stopHook)
	hooks := make([]Hook, len(opts.Hooks)+1)
	copy(hooks, opts.Hooks)
	hooks[len(opts.Hooks)] = sh

	state := opts.State
	if state == nil {
		state = new(State)
		state.init()
	}

	return &Context{
		diff:                opts.Diff,
		hooks:               hooks,
		module:              opts.Module,
		providers:           opts.Providers,
		providerInputConfig: make(map[string]map[string]interface{}),
		provisioners:        opts.Provisioners,
		sh:                  sh,
		state:               state,
		uiInput:             opts.UIInput,
		variables:           opts.Variables,
	}
}

// GraphBuilder returns the GraphBuilder that will be used to create
// the graphs for this context.
func (c *Context) GraphBuilder() GraphBuilder {
	// TODO test
	providers := make([]string, 0, len(c.providers))
	for k, _ := range c.providers {
		providers = append(providers, k)
	}

	provisioners := make([]string, 0, len(c.provisioners))
	for k, _ := range c.provisioners {
		provisioners = append(provisioners, k)
	}

	return &BuiltinGraphBuilder{
		Root:         c.module,
		Diff:         c.diff,
		Providers:    providers,
		Provisioners: provisioners,
		State:        c.state,
	}
}

// Input asks for input to fill variables and provider configurations.
// This modifies the configuration in-place, so asking for Input twice
// may result in different UI output showing different current values.
func (c *Context) Input(mode InputMode) error {
	v := c.acquireRun()
	defer c.releaseRun(v)

	if mode&InputModeVar != 0 {
		// Walk the variables first for the root module. We walk them in
		// alphabetical order for UX reasons.
		rootConf := c.module.Config()
		names := make([]string, len(rootConf.Variables))
		m := make(map[string]*config.Variable)
		for i, v := range rootConf.Variables {
			names[i] = v.Name
			m[v.Name] = v
		}
		sort.Strings(names)
		for _, n := range names {
			v := m[n]
			switch v.Type() {
			case config.VariableTypeMap:
				continue
			case config.VariableTypeString:
				// Good!
			default:
				panic(fmt.Sprintf("Unknown variable type: %#v", v.Type()))
			}

			var defaultString string
			if v.Default != nil {
				defaultString = v.Default.(string)
			}

			// Ask the user for a value for this variable
			var value string
			for {
				var err error
				value, err = c.uiInput.Input(&InputOpts{
					Id:          fmt.Sprintf("var.%s", n),
					Query:       fmt.Sprintf("var.%s", n),
					Default:     defaultString,
					Description: v.Description,
				})
				if err != nil {
					return fmt.Errorf(
						"Error asking for %s: %s", n, err)
				}

				if value == "" && v.Required() {
					// Redo if it is required.
					continue
				}

				if value == "" {
					// No value, just exit the loop. With no value, we just
					// use whatever is currently set in variables.
					break
				}

				break
			}

			if value != "" {
				c.variables[n] = value
			}
		}
	}

	if mode&InputModeProvider != 0 {
		// Do the walk
		if _, err := c.walk(walkInput); err != nil {
			return err
		}
	}

	return nil
}

// Apply applies the changes represented by this context and returns
// the resulting state.
//
// In addition to returning the resulting state, this context is updated
// with the latest state.
func (c *Context) Apply() (*State, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	// Copy our own state
	c.state = c.state.deepcopy()

	// Do the walk
	_, err := c.walk(walkApply)

	// Clean out any unused things
	c.state.prune()
	println(fmt.Sprintf("%#v", c.state))

	return c.state, err
}

// Plan generates an execution plan for the given context.
//
// The execution plan encapsulates the context and can be stored
// in order to reinstantiate a context later for Apply.
//
// Plan also updates the diff of this context to be the diff generated
// by the plan, so Apply can be called after.
func (c *Context) Plan(opts *PlanOpts) (*Plan, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	p := &Plan{
		Module: c.module,
		Vars:   c.variables,
		State:  c.state,
	}

	var operation walkOperation
	if opts != nil && opts.Destroy {
		operation = walkPlanDestroy
	} else {
		// Set our state to be something temporary. We do this so that
		// the plan can update a fake state so that variables work, then
		// we replace it back with our old state.
		old := c.state
		if old == nil {
			c.state = &State{}
			c.state.init()
		} else {
			c.state = old.deepcopy()
		}
		defer func() {
			c.state = old
		}()

		operation = walkPlan
	}

	// Setup our diff
	c.diffLock.Lock()
	c.diff = new(Diff)
	c.diff.init()
	c.diffLock.Unlock()

	// Do the walk
	if _, err := c.walk(operation); err != nil {
		return nil, err
	}
	p.Diff = c.diff

	return p, nil
}

// Refresh goes through all the resources in the state and refreshes them
// to their latest state. This will update the state that this context
// works with, along with returning it.
//
// Even in the case an error is returned, the state will be returned and
// will potentially be partially updated.
func (c *Context) Refresh() (*State, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	// Copy our own state
	c.state = c.state.deepcopy()

	// Do the walk
	if _, err := c.walk(walkRefresh); err != nil {
		return nil, err
	}

	// Clean out any unused things
	c.state.prune()

	return c.state, nil
}

// Stop stops the running task.
//
// Stop will block until the task completes.
func (c *Context) Stop() {
	c.l.Lock()
	ch := c.runCh

	// If we aren't running, then just return
	if ch == nil {
		c.l.Unlock()
		return
	}

	// Tell the hook we want to stop
	c.sh.Stop()

	// Wait for us to stop
	c.l.Unlock()
	<-ch
}

// Validate validates the configuration and returns any warnings or errors.
func (c *Context) Validate() ([]string, []error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	var errs error

	// Validate the configuration itself
	if err := c.module.Validate(); err != nil {
		errs = multierror.Append(errs, err)
	}

	// This only needs to be done for the root module, since inter-module
	// variables are validated in the module tree.
	if config := c.module.Config(); config != nil {
		// Validate the user variables
		if err := smcUserVariables(config, c.variables); len(err) > 0 {
			errs = multierror.Append(errs, err...)
		}
	}

	// Walk
	walker, err := c.walk(walkValidate)
	if err != nil {
		return nil, multierror.Append(errs, err).Errors
	}

	// Return the result
	rerrs := multierror.Append(errs, walker.ValidationErrors...)
	return walker.ValidationWarnings, rerrs.Errors
}

func (c *Context) acquireRun() chan<- struct{} {
	c.l.Lock()
	defer c.l.Unlock()

	// Wait for no channel to exist
	for c.runCh != nil {
		c.l.Unlock()
		ch := c.runCh
		<-ch
		c.l.Lock()
	}

	ch := make(chan struct{})
	c.runCh = ch
	return ch
}

func (c *Context) releaseRun(ch chan<- struct{}) {
	c.l.Lock()
	defer c.l.Unlock()

	close(ch)
	c.runCh = nil
	c.sh.Reset()
}

func (c *Context) walk(operation walkOperation) (*ContextGraphWalker, error) {
	// Build the graph
	graph, err := c.GraphBuilder().Build(RootModulePath)
	if err != nil {
		return nil, err
	}

	// Walk the graph
	walker := &ContextGraphWalker{Context: c, Operation: operation}
	return walker, graph.Walk(walker)
}

// walkOperation is an enum which tells the walkContext what to do.
type walkOperation byte

const (
	walkInvalid walkOperation = iota
	walkInput
	walkApply
	walkPlan
	walkPlanDestroy
	walkRefresh
	walkValidate
)
<commit_msg>terraform: Graph returns *Graph for now<commit_after>package terraform

import (
	"fmt"
	"sort"
	"sync"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/config/module"
)

// InputMode defines what sort of input will be asked for when Input
// is called on Context.
type InputMode byte

const (
	// InputModeVar asks for variables
	InputModeVar InputMode = 1 << iota

	// InputModeProvider asks for provider variables
	InputModeProvider

	// InputModeStd is the standard operating mode and asks for both variables
	// and providers.
	InputModeStd = InputModeVar | InputModeProvider
)

// ContextOpts are the user-configurable options to create a context with
// NewContext.
type ContextOpts struct {
	Diff         *Diff
	Hooks        []Hook
	Module       *module.Tree
	Parallelism  int
	State        *State
	Providers    map[string]ResourceProviderFactory
	Provisioners map[string]ResourceProvisionerFactory
	Variables    map[string]string

	UIInput UIInput
}

// Context represents all the context that Terraform needs in order to
// perform operations on infrastructure. This structure is built using
// NewContext. See the documentation for that.
type Context struct {
	diff         *Diff
	diffLock     sync.RWMutex
	hooks        []Hook
	module       *module.Tree
	providers    map[string]ResourceProviderFactory
	provisioners map[string]ResourceProvisionerFactory
	sh           *stopHook
	state        *State
	stateLock    sync.RWMutex
	uiInput      UIInput
	variables    map[string]string

	l                   sync.Mutex // Lock acquired during any task
	providerInputConfig map[string]map[string]interface{}
	runCh               <-chan struct{}
}

// NewContext creates a new Context structure.
//
// Once a Context is creator, the pointer values within ContextOpts
// should not be mutated in any way, since the pointers are copied, not
// the values themselves.
func NewContext(opts *ContextOpts) *Context {
	// Copy all the hooks and add our stop hook. We don't append directly
	// to the Config so that we're not modifying that in-place.
	sh := new(stopHook)
	hooks := make([]Hook, len(opts.Hooks)+1)
	copy(hooks, opts.Hooks)
	hooks[len(opts.Hooks)] = sh

	state := opts.State
	if state == nil {
		state = new(State)
		state.init()
	}

	return &Context{
		diff:                opts.Diff,
		hooks:               hooks,
		module:              opts.Module,
		providers:           opts.Providers,
		providerInputConfig: make(map[string]map[string]interface{}),
		provisioners:        opts.Provisioners,
		sh:                  sh,
		state:               state,
		uiInput:             opts.UIInput,
		variables:           opts.Variables,
	}
}

// Graph returns the graph for this config.
func (c *Context) Graph() (*Graph, error) {
	return c.GraphBuilder().Build(RootModulePath)
}

// GraphBuilder returns the GraphBuilder that will be used to create
// the graphs for this context.
func (c *Context) GraphBuilder() GraphBuilder {
	// TODO test
	providers := make([]string, 0, len(c.providers))
	for k, _ := range c.providers {
		providers = append(providers, k)
	}

	provisioners := make([]string, 0, len(c.provisioners))
	for k, _ := range c.provisioners {
		provisioners = append(provisioners, k)
	}

	return &BuiltinGraphBuilder{
		Root:         c.module,
		Diff:         c.diff,
		Providers:    providers,
		Provisioners: provisioners,
		State:        c.state,
	}
}

// Input asks for input to fill variables and provider configurations.
// This modifies the configuration in-place, so asking for Input twice
// may result in different UI output showing different current values.
func (c *Context) Input(mode InputMode) error {
	v := c.acquireRun()
	defer c.releaseRun(v)

	if mode&InputModeVar != 0 {
		// Walk the variables first for the root module. We walk them in
		// alphabetical order for UX reasons.
		rootConf := c.module.Config()
		names := make([]string, len(rootConf.Variables))
		m := make(map[string]*config.Variable)
		for i, v := range rootConf.Variables {
			names[i] = v.Name
			m[v.Name] = v
		}
		sort.Strings(names)
		for _, n := range names {
			v := m[n]
			switch v.Type() {
			case config.VariableTypeMap:
				continue
			case config.VariableTypeString:
				// Good!
			default:
				panic(fmt.Sprintf("Unknown variable type: %#v", v.Type()))
			}

			var defaultString string
			if v.Default != nil {
				defaultString = v.Default.(string)
			}

			// Ask the user for a value for this variable
			var value string
			for {
				var err error
				value, err = c.uiInput.Input(&InputOpts{
					Id:          fmt.Sprintf("var.%s", n),
					Query:       fmt.Sprintf("var.%s", n),
					Default:     defaultString,
					Description: v.Description,
				})
				if err != nil {
					return fmt.Errorf(
						"Error asking for %s: %s", n, err)
				}

				if value == "" && v.Required() {
					// Redo if it is required.
					continue
				}

				if value == "" {
					// No value, just exit the loop. With no value, we just
					// use whatever is currently set in variables.
					break
				}

				break
			}

			if value != "" {
				c.variables[n] = value
			}
		}
	}

	if mode&InputModeProvider != 0 {
		// Do the walk
		if _, err := c.walk(walkInput); err != nil {
			return err
		}
	}

	return nil
}

// Apply applies the changes represented by this context and returns
// the resulting state.
//
// In addition to returning the resulting state, this context is updated
// with the latest state.
func (c *Context) Apply() (*State, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	// Copy our own state
	c.state = c.state.deepcopy()

	// Do the walk
	_, err := c.walk(walkApply)

	// Clean out any unused things
	c.state.prune()
	println(fmt.Sprintf("%#v", c.state))

	return c.state, err
}

// Plan generates an execution plan for the given context.
//
// The execution plan encapsulates the context and can be stored
// in order to reinstantiate a context later for Apply.
//
// Plan also updates the diff of this context to be the diff generated
// by the plan, so Apply can be called after.
func (c *Context) Plan(opts *PlanOpts) (*Plan, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	p := &Plan{
		Module: c.module,
		Vars:   c.variables,
		State:  c.state,
	}

	var operation walkOperation
	if opts != nil && opts.Destroy {
		operation = walkPlanDestroy
	} else {
		// Set our state to be something temporary. We do this so that
		// the plan can update a fake state so that variables work, then
		// we replace it back with our old state.
		old := c.state
		if old == nil {
			c.state = &State{}
			c.state.init()
		} else {
			c.state = old.deepcopy()
		}
		defer func() {
			c.state = old
		}()

		operation = walkPlan
	}

	// Setup our diff
	c.diffLock.Lock()
	c.diff = new(Diff)
	c.diff.init()
	c.diffLock.Unlock()

	// Do the walk
	if _, err := c.walk(operation); err != nil {
		return nil, err
	}
	p.Diff = c.diff

	return p, nil
}

// Refresh goes through all the resources in the state and refreshes them
// to their latest state. This will update the state that this context
// works with, along with returning it.
//
// Even in the case an error is returned, the state will be returned and
// will potentially be partially updated.
func (c *Context) Refresh() (*State, error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	// Copy our own state
	c.state = c.state.deepcopy()

	// Do the walk
	if _, err := c.walk(walkRefresh); err != nil {
		return nil, err
	}

	// Clean out any unused things
	c.state.prune()

	return c.state, nil
}

// Stop stops the running task.
//
// Stop will block until the task completes.
func (c *Context) Stop() {
	c.l.Lock()
	ch := c.runCh

	// If we aren't running, then just return
	if ch == nil {
		c.l.Unlock()
		return
	}

	// Tell the hook we want to stop
	c.sh.Stop()

	// Wait for us to stop
	c.l.Unlock()
	<-ch
}

// Validate validates the configuration and returns any warnings or errors.
func (c *Context) Validate() ([]string, []error) {
	v := c.acquireRun()
	defer c.releaseRun(v)

	var errs error

	// Validate the configuration itself
	if err := c.module.Validate(); err != nil {
		errs = multierror.Append(errs, err)
	}

	// This only needs to be done for the root module, since inter-module
	// variables are validated in the module tree.
	if config := c.module.Config(); config != nil {
		// Validate the user variables
		if err := smcUserVariables(config, c.variables); len(err) > 0 {
			errs = multierror.Append(errs, err...)
		}
	}

	// Walk
	walker, err := c.walk(walkValidate)
	if err != nil {
		return nil, multierror.Append(errs, err).Errors
	}

	// Return the result
	rerrs := multierror.Append(errs, walker.ValidationErrors...)
	return walker.ValidationWarnings, rerrs.Errors
}

func (c *Context) acquireRun() chan<- struct{} {
	c.l.Lock()
	defer c.l.Unlock()

	// Wait for no channel to exist
	for c.runCh != nil {
		c.l.Unlock()
		ch := c.runCh
		<-ch
		c.l.Lock()
	}

	ch := make(chan struct{})
	c.runCh = ch
	return ch
}

func (c *Context) releaseRun(ch chan<- struct{}) {
	c.l.Lock()
	defer c.l.Unlock()

	close(ch)
	c.runCh = nil
	c.sh.Reset()
}

func (c *Context) walk(operation walkOperation) (*ContextGraphWalker, error) {
	// Build the graph
	graph, err := c.GraphBuilder().Build(RootModulePath)
	if err != nil {
		return nil, err
	}

	// Walk the graph
	walker := &ContextGraphWalker{Context: c, Operation: operation}
	return walker, graph.Walk(walker)
}

// walkOperation is an enum which tells the walkContext what to do.
type walkOperation byte

const (
	walkInvalid walkOperation = iota
	walkInput
	walkApply
	walkPlan
	walkPlanDestroy
	walkRefresh
	walkValidate
)
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.10.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>v0.10.0-rc1<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.10.0-rc1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
var VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>package itest

import (
	"testing"
	"github.com/stretchr/testify/assert"
	"net/http"
)

func TestETE(t *testing.T) {

	res, err := SendEvent(Push, "nginx", "master-default-nginxdeployment-1.10")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)
}
<commit_msg>skip integration tests unless adding a dedicated env. var;<commit_after>package itest

import (
	"testing"
	"github.com/stretchr/testify/assert"
	"net/http"
	"os"
)

func TestETE(t *testing.T) {

	const itest = "INTEGRATION_TESTS"
	if os.Getenv(itest) != "true" {
		t.Skipf("TestETE skiped. To run it, please, add enviroment varible: %s=true", itest)
	}

	res, err := SendEvent(Push, "nginx", "master-default-nginxdeployment-1.10")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)
}
<|endoftext|>


<commit_before>// Package ledstrip has control routines for LED strips
// Author: Tim Scheuermann (https://github.com/noxer)
package ledstrip

import (
	"github.com/noxer/tinkerforge"
	"github.com/noxer/tinkerforge/helpers"
)

// LedStrip is a control structure for LED strips
type LedStrip struct {
	t           tinkerforge.Tinkerforge
	uid         uint32
	colorMap    [3]int
	revColorMap [3]int
}

// Color represents a three byte value (8 bit for red, green and blue respectively).
type Color [3]byte

var (
	// ColorMapRGB represents the standard color mapping (R->R, G->G, B->B)
	ColorMapRGB = [3]int{0, 1, 2}
	// ColorMapBGR represents the color mapping needed when red and blue are flipped
	ColorMapBGR = [3]int{2, 1, 0}
)

// ChipType represents different types of control chips.
type ChipType uint16

const (
	// WS2801 chip type ID
	WS2801 ChipType = 2801
	// WS2811 chip type ID
	WS2811 = 2811
	// WS2812 chip type ID
	WS2812 = 2812
)

// New creates a new LED strip control for the bricklet with 'uid'.
func New(t tinkerforge.Tinkerforge, uid uint32) *LedStrip {
	return &LedStrip{
		t:           t,
		uid:         uid,
		colorMap:    [3]int{0, 1, 2},
		revColorMap: [3]int{0, 1, 2},
	}
}

// SetRGBValues sets up to 16 color values beginning from 'index' to the values in 'colors'.
func (l *LedStrip) SetRGBValues(index uint16, colors []*Color) error {
	// The rgb data
	r, g, b := [16]byte{}, [16]byte{}, [16]byte{}

	// Trim the slice if necessary
	if len(colors) > 16 {
		colors = colors[:16]
	}

	// Copy the colors into the arrays, apply color mapping
	for i, c := range colors {
		r[i] = c[l.colorMap[0]]
		g[i] = c[l.colorMap[1]]
		b[i] = c[l.colorMap[2]]
	}

	// Create packet
	p, err := tinkerforge.NewPacket(l.uid, 1, false, index, uint8(len(colors)), r, g, b)
	if err != nil {
		return err
	}

	// Send packet
	_, err = l.t.Send(p)
	return err
}

// GetRGBValues retrieves the currently set RGB values of the LED strip beginning from 'index' and up to 'length' values.
func (l *LedStrip) GetRGBValues(index uint16, length uint8) ([]*Color, error) {
	// Limit the length to 16 (maximum the protocol supports)
	if length > 16 {
		length = 16
	}

	// Create a new tinkerforge packet for function #2
	p, err := tinkerforge.NewPacket(l.uid, 2, true, index, length)
	if err != nil {
		return nil, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return nil, err
	}

	// Decode the values from the answer
	r, g, b := [16]byte{}, [16]byte{}, [16]byte{}
	if err = res.Decode(&r, &g, &b); err != nil {
		return nil, err
	}

	// Transform the single color arrays into color values
	result := make([]*Color, length)
	for i := 0; i < int(length); i++ {
		result[i] = &Color{}
		result[i][l.revColorMap[0]] = r[i]
		result[i][l.revColorMap[1]] = g[i]
		result[i][l.revColorMap[2]] = b[i]
	}

	return result, nil
}

// SetFrameDuration sets the number of milliseconds between frames.
func (l *LedStrip) SetFrameDuration(ms uint16) error {
	// Create a new tinkerforge packet for function #3
	p, err := tinkerforge.NewPacket(l.uid, 3, false, ms)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetFrameDuration returns the currently set number of milliseconds between frames.
func (l *LedStrip) GetFrameDuration() (uint16, error) {
	// Create a tinkerforge packet for function #4
	p, err := tinkerforge.NewPacket(l.uid, 4, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the duration
	var duration uint16
	if err = res.Decode(&duration); err != nil {
		return 0, err
	}

	return duration, nil
}

// GetSupplyVoltage returns the current voltage the LED strip's LEDs consume in mV.
func (l *LedStrip) GetSupplyVoltage() (uint16, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 5, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the voltage
	var voltage uint16
	if err = res.Decode(&voltage); err != nil {
		return 0, err
	}

	return voltage, nil
}

// SetClockFrequency sets the frequency of the clock in hertz.
// Allowed values range from 10000 (10kHz) to 2000000 (2MHz).
// The bricklet chooses the next possible frequency automatically.
func (l *LedStrip) SetClockFrequency(frequency uint32) error {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 7, false, frequency)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetClockFrequency returns the currently used clock frequency.
func (l *LedStrip) GetClockFrequency() (uint32, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 8, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the frequency
	var frequency uint32
	if err = res.Decode(&frequency); err != nil {
		return 0, err
	}

	return frequency, nil
}

// SetChipType sets the type of the LEDs control chip.
func (l *LedStrip) SetChipType(chipType ChipType) error {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 9, false, chipType)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetChipType returns the currently set type of the LEDs control chip.
func (l *LedStrip) GetChipType() (ChipType, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 10, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the chip type
	var chipType ChipType
	if err = res.Decode(&chipType); err != nil {
		return 0, err
	}

	return chipType, nil
}

// GetIdentity returns the position information of the bricklet and its identifier.
func (l *LedStrip) GetIdentity() (*helpers.BrickletIdentity, error) {
	// Call the helper function for getting the identity
	i, err := helpers.GetIdentity(l.t, l.uid)
	return i, err
}

type frameRenderedHandler func(uint16)

func (f frameRenderedHandler) Handle(p *tinkerforge.Packet) {

	var length uint16

	if p.Decode(&length) != nil {
		return
	}
	f(length)

}

// CallbackFrameRendered is a convenience function for registering
// a handler to be called when a frame has been rendered.
func (l *LedStrip) CallbackFrameRendered(handler func(uint16)) {

	if handler == nil {
		l.t.Handler(l.uid, 6, nil)
	} else {
		l.t.Handler(l.uid, 6, frameRenderedHandler(handler))
	}

}

// SetColorMapping sets a color mapping to be applied when getting or setting RGB values.
func (l *LedStrip) SetColorMapping(mapping [3]int) {
	// Set the color map
	l.colorMap = mapping

	// Calculate the reverse map (for GetRGBValues)
	for i, v := range mapping {
		l.revColorMap[v] = i
	}

}
<commit_msg>minor API changes<commit_after>// Package ledstrip has control routines for LED strips
// Author: Tim Scheuermann (https://github.com/noxer)
package ledstrip

import (
	"github.com/noxer/tinkerforge"
	"github.com/noxer/tinkerforge/helpers"
)

// LedStrip is a control structure for LED strips
type LedStrip struct {
	t           tinkerforge.Tinkerforge
	uid         uint32
	colorMap    [3]int
	revColorMap [3]int
}

// Color represents a three byte value (8 bit for red, green and blue respectively).
type Color [3]byte

var (
	// ColorMapRGB represents the standard color mapping (R->R, G->G, B->B)
	ColorMapRGB = [3]int{0, 1, 2}
	// ColorMapBGR represents the color mapping needed when red and blue are flipped
	ColorMapBGR = [3]int{2, 1, 0}
)

// ChipType represents different types of control chips.
type ChipType uint16

const (
	// WS2801 chip type ID
	WS2801 ChipType = 2801
	// WS2811 chip type ID
	WS2811 = 2811
	// WS2812 chip type ID
	WS2812 = 2812
)

// New creates a new LED strip control for the bricklet with 'uid'.
func New(t tinkerforge.Tinkerforge, uid string) (*LedStrip, error) {
	readUID, err := helpers.Base58ToU32(uid)
	if err != nil {
		return nil, err
	}
	return &LedStrip{
		t:           t,
		uid:         readUID,
		colorMap:    [3]int{0, 1, 2},
		revColorMap: [3]int{0, 1, 2},
	}, nil
}

// SetRGBValues sets up to 16 color values beginning from 'index' to the values in 'colors'.
func (l *LedStrip) SetRGBValues(index uint16, colors []Color) error {
	// The rgb data
	r, g, b := [16]byte{}, [16]byte{}, [16]byte{}

	// Trim the slice if necessary
	if len(colors) > 16 {
		colors = colors[:16]
	}

	// Copy the colors into the arrays, apply color mapping
	for i, c := range colors {
		r[i] = c[l.colorMap[0]]
		g[i] = c[l.colorMap[1]]
		b[i] = c[l.colorMap[2]]
	}

	// Create packet
	p, err := tinkerforge.NewPacket(l.uid, 1, false, index, uint8(len(colors)), r, g, b)
	if err != nil {
		return err
	}

	// Send packet
	_, err = l.t.Send(p)
	return err
}

// GetRGBValues retrieves the currently set RGB values of the LED strip beginning from 'index' and up to 'length' values.
func (l *LedStrip) GetRGBValues(index uint16, length uint8) ([]Color, error) {
	// Limit the length to 16 (maximum the protocol supports)
	if length > 16 {
		length = 16
	}

	// Create a new tinkerforge packet for function #2
	p, err := tinkerforge.NewPacket(l.uid, 2, true, index, length)
	if err != nil {
		return nil, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return nil, err
	}

	// Decode the values from the answer
	r, g, b := [16]byte{}, [16]byte{}, [16]byte{}
	if err = res.Decode(&r, &g, &b); err != nil {
		return nil, err
	}

	// Transform the single color arrays into color values
	result := make([]Color, length)
	for i := 0; i < int(length); i++ {
		result[i][l.revColorMap[0]] = r[i]
		result[i][l.revColorMap[1]] = g[i]
		result[i][l.revColorMap[2]] = b[i]
	}

	return result, nil
}

// SetFrameDuration sets the number of milliseconds between frames.
func (l *LedStrip) SetFrameDuration(ms uint16) error {
	// Create a new tinkerforge packet for function #3
	p, err := tinkerforge.NewPacket(l.uid, 3, false, ms)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetFrameDuration returns the currently set number of milliseconds between frames.
func (l *LedStrip) GetFrameDuration() (uint16, error) {
	// Create a tinkerforge packet for function #4
	p, err := tinkerforge.NewPacket(l.uid, 4, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the duration
	var duration uint16
	if err = res.Decode(&duration); err != nil {
		return 0, err
	}

	return duration, nil
}

// GetSupplyVoltage returns the current voltage the LED strip's LEDs consume in mV.
func (l *LedStrip) GetSupplyVoltage() (uint16, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 5, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the voltage
	var voltage uint16
	if err = res.Decode(&voltage); err != nil {
		return 0, err
	}

	return voltage, nil
}

// SetClockFrequency sets the frequency of the clock in hertz.
// Allowed values range from 10000 (10kHz) to 2000000 (2MHz).
// The bricklet chooses the next possible frequency automatically.
func (l *LedStrip) SetClockFrequency(frequency uint32) error {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 7, false, frequency)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetClockFrequency returns the currently used clock frequency.
func (l *LedStrip) GetClockFrequency() (uint32, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 8, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the frequency
	var frequency uint32
	if err = res.Decode(&frequency); err != nil {
		return 0, err
	}

	return frequency, nil
}

// SetChipType sets the type of the LEDs control chip.
func (l *LedStrip) SetChipType(chipType ChipType) error {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 9, false, chipType)
	if err != nil {
		return err
	}

	// Send the packet
	_, err = l.t.Send(p)
	return err

}

// GetChipType returns the currently set type of the LEDs control chip.
func (l *LedStrip) GetChipType() (ChipType, error) {
	// Create a new tinkerforge packet
	p, err := tinkerforge.NewPacket(l.uid, 10, true)
	if err != nil {
		return 0, err
	}

	// Send the packet
	res, err := l.t.Send(p)
	if err != nil {
		return 0, err
	}

	// Decode the chip type
	var chipType ChipType
	if err = res.Decode(&chipType); err != nil {
		return 0, err
	}

	return chipType, nil
}

// GetIdentity returns the position information of the bricklet and its identifier.
func (l *LedStrip) GetIdentity() (*helpers.BrickletIdentity, error) {
	// Call the helper function for getting the identity
	i, err := helpers.GetIdentity(l.t, l.uid)
	return i, err
}

type frameRenderedHandler func(uint16)

func (f frameRenderedHandler) Handle(p *tinkerforge.Packet) {

	var length uint16

	if p.Decode(&length) != nil {
		return
	}
	f(length)

}

// CallbackFrameRendered is a convenience function for registering
// a handler to be called when a frame has been rendered.
func (l *LedStrip) CallbackFrameRendered(handler func(uint16)) {

	if handler == nil {
		l.t.Handler(l.uid, 6, nil)
	} else {
		l.t.Handler(l.uid, 6, frameRenderedHandler(handler))
	}

}

// SetColorMapping sets a color mapping to be applied when getting or setting RGB values.
func (l *LedStrip) SetColorMapping(mapping [3]int) {
	// Set the color map
	l.colorMap = mapping

	// Calculate the reverse map (for GetRGBValues)
	for i, v := range mapping {
		l.revColorMap[v] = i
	}

}
<|endoftext|>


<commit_before>package terp

import (
	"bytes"
	. "fmt"
	"log"
)

func White(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n'
}

func WhiteNotNewline(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r'
}

func WhiteOrSemi(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == ';'
}

func (me *Terp) Eval(s string) (result Any) {
	result = "" // In case there are no commands.
	log.Printf("< Eval < %#v\n", s)
	rest := s
Loop:
	for {
		var words List
		words, rest = me.ParseCmd(rest)
		if len(words) == 0 {
			break Loop
		}
		result = me.Apply(words)
	}
	if len(rest) > 0 {
		panic(Sprintf("Eval: Did not eval entire string: rest=<%q>", rest))
	}
	log.Printf("> Eval > %#v\n", result)
	return
}

// Parse nested curlies, returning contents and new position
func (me *Terp) ParseCurly(s string) (result Any, rest string) {
	Must('{', s[0], "ParseCurly should begin at open curly")
	n := len(s)
	i := 1

	buf := bytes.NewBuffer(nil)
	c := s[i]
	b := 1 // brace depth
Loop:
	for i < n {
		c = s[i]
		switch c {
		case '{':
			b++
		case '}':
			b--
		}
		if b == 0 {
			break Loop
		}
		buf.WriteByte(c)
		i++
	}
	if c != '}' {
		panic("ParseCurly: missing end brace:" + Repr(c))
	}
	i++

	result = buf.String()
	rest = s[i:]
	return
}

// TODO: ParseSquare is too much like Eval.
// Parse Square Bracketed subcommand, returning result and new position
func (me *Terp) ParseSquare(s string) (result Any, rest string) {
	log.Printf("< ParseSquare < %#v\n", s)
	result = "" // In case there are no commands.
	Must(byte('['), byte(s[0]), "ParseSquare should begin at open square")
	rest = s[1:]

Loop:
	for {
		var words List
		words, rest = me.ParseCmd(rest)
		if len(words) == 0 {
			break Loop
		}
		result = me.Apply(words)
	}
	if len(rest) == 0 || rest[0] != ']' {
		panic("ParseSquare: missing end bracket")
	}
	rest = rest[1:]
	log.Printf("> ParseSquare > %#v > %q\n", result, rest)
	return
}

func (me *Terp) ParseQuote(s string) (result Any, rest string) {
	panic("ParseQuote: not yet")
}

// Parse a bareword, returning result and new position
func (me *Terp) ParseWord(s string) (result Any, rest string) {
	log.Printf("< ParseWord < %#v\n", s)
	i := 0
	n := len(s)
	buf := bytes.NewBuffer(nil)
Loop:
	for i < n {
		c := s[i]
		switch c {
		case '[':
			// Mid-word, squares should return stringlike result.
			result, rest := me.ParseSquare(s)
			buf.WriteString(Str(result))
			s = rest
			n = len(s)
			i = 0
		case ']':
			break Loop
		case ' ', '\t', '\n', '\r', ';':
			break Loop
		case '"':
			panic("ParseWord: DoubleQuote inside word")
		default:
			buf.WriteByte(c)
			i++
		}
	}
	result = buf.String()
	rest = s[i:]
	log.Printf("> ParseWord > %#v > %q\n", result, rest)
	return
}

// Might return nonempty <rest> if it finds ']'
// Returns next command as List (may be empty) (substituting as needed) and remaining string.
func (me *Terp) ParseCmd(str string) (z List, s string) {
	s = str
	log.Printf("< ParseCmd < %#v\n", s)
	z = make(List, 0, 8)
	var c uint8

	// skip space or ;
	i := 0
	n := len(s)
	for i < n {
		c = s[i]
		if !WhiteOrSemi(s[i]) {
			break
		}
		i++
	}
	s = s[i:]

Loop:
	for len(s) > 0 {
		log.Printf("* ParseCmd * TopLoop * z=%#v * s=%q\n", z, s)
		// found non-white
		switch s[0] {
		case ']':
			break Loop
		case '{':
			result, rest := me.ParseCurly(s)
			z = append(z, result)
			s = rest
		case '[':
			result, rest := me.ParseSquare(s)
			z = append(z, result)
			s = rest
		case '"':
			result, rest := me.ParseQuote(s)
			z = append(z, result)
			s = rest
		default:
			result, rest := me.ParseWord(s)
			z = append(z, result)
			s = rest
		}

		// skip white
		log.Printf("* ParseCmd * skip white * z=%#v * s=%q\n", z, s)
		n = len(s)
		i = 0
	Skip:
		for i < n {
			switch s[i] {
			case ' ', '\t', '\r':
				i++
				continue Skip
			case ';', '\n':
				break Skip
			default:
				break Skip
			}
		}
		s = s[i:]
		if len(s) == 0 {
			break Loop // end of string
		}
		c = s[0]
		if c == ';' || c == '\n' {
			s = s[1:]  // Omit the semicolon or newline
			break Loop // end of cmd
		}
		log.Printf("* ParseCmd * End Loop * z=%#v * s=%q\n", z, s)
	} // End Loop
	log.Printf("* ParseCmd * Break Loop * z=%#v * s=%q\n", z, s)

	log.Printf("> ParseCmd > %#v > %q\n", z, s)
	return
}

func ParseList(s string) List {
	n := len(s)
	i := 0
	z := make(List, 0, 8)

	for i < n {
		var c uint8

		// skip space
		for i < n {
			c = s[i]
			if !White(s[i]) {
				break
			}
			i++
		}
		if i == n {
			break
		}

		buf := bytes.NewBuffer(nil)

		// found non-white
		if c == '{' {
			i++
			c = s[i]
			b := 1
			for i < n {
				c = s[i]
				switch c {
				case '{':
					b++
				case '}':
					b--
				}
				if b == 0 {
					break
				}
				buf.WriteByte(c)
				i++
			}
			if c != '}' {
				panic("ParseList: missing end brace:" + Repr(c))
			}
			i++
		} else {
			for i < n {
				c = s[i]
				if White(s[i]) {
					break
				}
				buf.WriteByte(c)
				i++
			}
		}
		z = append(z, buf.String())
	}
	return z
}
<commit_msg>parser fixes<commit_after>package terp

import (
	"bytes"
	. "fmt"
	"log"
)

func White(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n'
}

func WhiteNotNewline(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r'
}

func WhiteOrSemi(ch uint8) bool {
	return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n' || ch == ';'
}

func (me *Terp) Eval(s string) (result Any) {
	result = "" // In case there are no commands.
	log.Printf("< Eval < %#v\n", s)
	rest := s
Loop:
	for {
		var words List
		words, rest = me.ParseCmd(rest)
		if len(words) == 0 {
			break Loop
		}
		result = me.Apply(words)
	}
	if len(rest) > 0 {
		panic(Sprintf("Eval: Did not eval entire string: rest=<%q>", rest))
	}
	log.Printf("> Eval > %#v\n", result)
	return
}

// Parse nested curlies, returning contents and new position
func (me *Terp) ParseCurly(s string) (result Any, rest string) {
	if s[0] != '{' {
		panic("ParseCurly should begin at open curly")
	} // '}'
	n := len(s)
	i := 1

	buf := bytes.NewBuffer(nil)
	c := s[i]
	b := 1 // brace depth
Loop:
	for i < n {
		c = s[i]
		switch c {
		case '{':
			b++
		case '}':
			b--
		}
		if b == 0 {
			break Loop
		}
		buf.WriteByte(c)
		i++
	}
	if c != '}' {
		panic("ParseCurly: missing end brace:" + Repr(c))
	}
	i++

	result = buf.String()
	rest = s[i:]
	return
}

// TODO: ParseSquare is too much like Eval.
// Parse Square Bracketed subcommand, returning result and new position
func (me *Terp) ParseSquare(s string) (result Any, rest string) {
	log.Printf("< ParseSquare < %#v\n", s)
	result = "" // In case there are no commands.
	if s[0] != '[' {
		panic("ParseSquare should begin at open square")
	}
	rest = s[1:]

Loop:
	for {
		var words List
		words, rest = me.ParseCmd(rest)
		if len(words) == 0 {
			break Loop
		}
		result = me.Apply(words)
	}
	if len(rest) == 0 || rest[0] != ']' {
		panic("ParseSquare: missing end bracket")
	}
	rest = rest[1:]
	log.Printf("> ParseSquare > %#v > %q\n", result, rest)
	return
}

func (me *Terp) ParseQuote(s string) (result Any, rest string) {
	log.Printf("< ParseQuote < %#v\n", s)
	if s[0] != '"' {
		panic("ParseCurly should begin at open curly")
	}
	i := 1
	n := len(s)
	buf := bytes.NewBuffer(nil)
Loop:
	for i < n {
		c := s[i]
		switch c {
		case '[':
			// Mid-word, squares should return stringlike result.
			result, rest := me.ParseSquare(s[i:])
			buf.WriteString(Str(result))
			s = rest
			n = len(s)
			i = 0
		case ']':
			panic("ParseQuote: CloseSquareBracket inside Quote")
		case '"':
			i++
			break Loop
		default:
			buf.WriteByte(c)
			i++
		}
	}
	result = buf.String()
	rest = s[i:]
	log.Printf("> ParseQuote > %#v > %q\n", result, rest)
	return
}

// Parse a bareword, returning result and new position
func (me *Terp) ParseWord(s string) (result Any, rest string) {
	log.Printf("< ParseWord < %#v\n", s)
	i := 0
	n := len(s)
	buf := bytes.NewBuffer(nil)
Loop:
	for i < n {
		c := s[i]
		switch c {
		case '[':
			// Mid-word, squares should return stringlike result.
			result, rest := me.ParseSquare(s)
			buf.WriteString(Str(result))
			s = rest
			n = len(s)
			i = 0
		case ']':
			break Loop
		case ' ', '\t', '\n', '\r', ';':
			break Loop
		case '"':
			panic("ParseWord: DoubleQuote inside word")
		default:
			buf.WriteByte(c)
			i++
		}
	}
	result = buf.String()
	rest = s[i:]
	log.Printf("> ParseWord > %#v > %q\n", result, rest)
	return
}

// Might return nonempty <rest> if it finds ']'
// Returns next command as List (may be empty) (substituting as needed) and remaining string.
func (me *Terp) ParseCmd(str string) (z List, s string) {
	s = str
	log.Printf("< ParseCmd < %#v\n", s)
	z = make(List, 0, 8)
	var c uint8

	// skip space or ;
	i := 0
	n := len(s)
	for i < n {
		c = s[i]
		if !WhiteOrSemi(s[i]) {
			break
		}
		i++
	}
	s = s[i:]

Loop:
	for len(s) > 0 {
		log.Printf("* ParseCmd * TopLoop * z=%#v * s=%q\n", z, s)
		// found non-white
		switch s[0] {
		case ']':
			break Loop
		case '{':
			result, rest := me.ParseCurly(s)
			z = append(z, result)
			s = rest
		case '[':
			result, rest := me.ParseSquare(s)
			z = append(z, result)
			s = rest
		case '"':
			result, rest := me.ParseQuote(s)
			z = append(z, result)
			s = rest
		default:
			result, rest := me.ParseWord(s)
			z = append(z, result)
			s = rest
		}

		// skip white
		log.Printf("* ParseCmd * skip white * z=%#v * s=%q\n", z, s)
		n = len(s)
		i = 0
	Skip:
		for i < n {
			switch s[i] {
			case ' ', '\t', '\r':
				i++
				continue Skip
			case ';', '\n':
				break Skip
			default:
				break Skip
			}
		}
		s = s[i:]
		if len(s) == 0 {
			break Loop // end of string
		}
		c = s[0]
		if c == ';' || c == '\n' {
			s = s[1:]  // Omit the semicolon or newline
			break Loop // end of cmd
		}
		log.Printf("* ParseCmd * End Loop * z=%#v * s=%q\n", z, s)
	} // End Loop
	log.Printf("* ParseCmd * Break Loop * z=%#v * s=%q\n", z, s)

	log.Printf("> ParseCmd > %#v > %q\n", z, s)
	return
}

func ParseList(s string) List {
	n := len(s)
	i := 0
	z := make(List, 0, 8)

	for i < n {
		var c uint8

		// skip space
		for i < n {
			c = s[i]
			if !White(s[i]) {
				break
			}
			i++
		}
		if i == n {
			break
		}

		buf := bytes.NewBuffer(nil)

		// found non-white
		if c == '{' {
			i++
			c = s[i]
			b := 1
			for i < n {
				c = s[i]
				switch c {
				case '{':
					b++
				case '}':
					b--
				}
				if b == 0 {
					break
				}
				buf.WriteByte(c)
				i++
			}
			if c != '}' {
				panic("ParseList: missing end brace:" + Repr(c))
			}
			i++
		} else {
			for i < n {
				c = s[i]
				if White(s[i]) {
					break
				}
				buf.WriteByte(c)
				i++
			}
		}
		z = append(z, buf.String())
	}
	return z
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
// Copyright (c) 2017 Hiraaki Nakamura
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ltsv

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/uber-go/zap"
)

const (
	// For JSON-escaping; see jsonEncoder.safeAddString below.
	_hex = "0123456789abcdef"
	// Initial buffer size for encoders.
	_initialBufSize = 1024
)

var (
	// errNilSink signals that Encoder.WriteEntry was called with a nil WriteSyncer.
	errNilSink = errors.New("can't write encoded message a nil WriteSyncer")
)

// Replacer is the interface that replaces a list of strings with replacements.
type Replacer interface {
	// Replace returns a copy of s with all replacements performed.
	Replace(s string) string
}

var ltsvPool = sync.Pool{New: func() interface{} {
	return &ltsvEncoder{
		bytes:        make([]byte, 0, _initialBufSize),
		timeLabel:    "time",
		levelLabel:   "level",
		messageLabel: "msg",
		replacer: strings.NewReplacer(
			"\t", "\\t",
			"\n", "\\n",
			"\r", "\\r",
		),
	}
}}

type ltsvEncoder struct {
	bytes        []byte
	timeLabel    string
	timeFmt      string
	levelLabel   string
	messageLabel string
	nestedLevel  int
	replacer     Replacer
}

// NewLTSVEncoder creates a line-oriented LTSV encoder.
// See http://ltsv.org/ for LTSV (Labeled Tab-separated Values).
// By default, the encoder uses RFC3339-formatted timestamps.
func NewLTSVEncoder(options ...LTSVOption) zap.Encoder {
	enc := ltsvPool.Get().(*ltsvEncoder)
	enc.truncate()
	enc.timeFmt = time.RFC3339
	for _, opt := range options {
		opt.apply(enc)
	}
	return enc
}

func (enc *ltsvEncoder) Free() {
	ltsvPool.Put(enc)
}

func (enc *ltsvEncoder) AddString(key, val string) {
	enc.addKey(key)
	if enc.nestedLevel > 0 {
		enc.bytes = append(enc.bytes, '"')
		enc.safeAddJSONString(val)
		enc.bytes = append(enc.bytes, '"')
	} else {
		enc.safeAddString(val)
	}
}

func (enc *ltsvEncoder) AddBool(key string, val bool) {
	enc.addKey(key)
	enc.bytes = strconv.AppendBool(enc.bytes, val)
}

func (enc *ltsvEncoder) AddInt(key string, val int) {
	enc.AddInt64(key, int64(val))
}

func (enc *ltsvEncoder) AddInt64(key string, val int64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendInt(enc.bytes, val, 10)
}

func (enc *ltsvEncoder) AddUint(key string, val uint) {
	enc.AddUint64(key, uint64(val))
}

func (enc *ltsvEncoder) AddUint64(key string, val uint64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendUint(enc.bytes, val, 10)
}

func (enc *ltsvEncoder) AddUintptr(key string, val uintptr) {
	enc.addKey(key)
	enc.bytes = append(enc.bytes, "0x"...)
	enc.bytes = strconv.AppendUint(enc.bytes, uint64(val), 16)
}

func (enc *ltsvEncoder) AddFloat64(key string, val float64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendFloat(enc.bytes, val, 'f', -1, 64)
}

func (enc *ltsvEncoder) AddMarshaler(key string, obj zap.LogMarshaler) error {
	enc.addKey(key)
	enc.nestedLevel++
	enc.bytes = append(enc.bytes, '{')
	err := obj.MarshalLog(enc)
	enc.bytes = append(enc.bytes, '}')
	enc.nestedLevel--
	return err
}

func (enc *ltsvEncoder) AddObject(key string, obj interface{}) error {
	enc.AddString(key, fmt.Sprintf("%+v", obj))
	return nil
}

func (enc *ltsvEncoder) Clone() zap.Encoder {
	clone := ltsvPool.Get().(*ltsvEncoder)
	clone.truncate()
	clone.bytes = append(clone.bytes, enc.bytes...)
	clone.timeLabel = enc.timeLabel
	clone.timeFmt = enc.timeFmt
	clone.levelLabel = enc.levelLabel
	clone.messageLabel = enc.messageLabel
	clone.nestedLevel = enc.nestedLevel
	clone.replacer = enc.replacer
	return clone
}

func (enc *ltsvEncoder) WriteEntry(sink io.Writer, msg string, lvl zap.Level, t time.Time) error {
	if sink == nil {
		return errNilSink
	}

	final := ltsvPool.Get().(*ltsvEncoder)
	final.truncate()
	enc.addTime(final, t)
	enc.addLevel(final, lvl)
	enc.addMessage(final, msg)

	if len(enc.bytes) > 0 {
		final.bytes = append(final.bytes, '\t')
		final.bytes = append(final.bytes, enc.bytes...)
	}
	final.bytes = append(final.bytes, '\n')

	expectedBytes := len(final.bytes)
	n, err := sink.Write(final.bytes)
	final.Free()
	if err != nil {
		return err
	}
	if n != expectedBytes {
		return fmt.Errorf("incomplete write: only wrote %v of %v bytes", n, expectedBytes)
	}
	return nil
}

func (enc *ltsvEncoder) truncate() {
	enc.bytes = enc.bytes[:0]
}

func (enc *ltsvEncoder) addKey(key string) {
	if enc.nestedLevel > 0 {
		last := len(enc.bytes) - 1
		// At some point, we'll also want to support arrays.
		if last >= 0 && enc.bytes[last] != '{' {
			enc.bytes = append(enc.bytes, ',')
		}
		enc.bytes = append(enc.bytes, '"')
		enc.safeAddJSONString(key)
		enc.bytes = append(enc.bytes, '"', ':')
	} else {
		if len(enc.bytes) > 0 {
			enc.bytes = append(enc.bytes, '\t')
		}
		enc.safeAddString(key)
		enc.bytes = append(enc.bytes, ':')
	}
}

func (enc *ltsvEncoder) addLevel(final *ltsvEncoder, lvl zap.Level) {
	final.addKey(enc.levelLabel)
	switch lvl {
	case zap.DebugLevel:
		final.bytes = append(final.bytes, 'D')
	case zap.InfoLevel:
		final.bytes = append(final.bytes, 'I')
	case zap.WarnLevel:
		final.bytes = append(final.bytes, 'W')
	case zap.ErrorLevel:
		final.bytes = append(final.bytes, 'E')
	case zap.PanicLevel:
		final.bytes = append(final.bytes, 'P')
	case zap.FatalLevel:
		final.bytes = append(final.bytes, 'F')
	default:
		final.bytes = strconv.AppendInt(final.bytes, int64(lvl), 10)
	}
}

func (enc *ltsvEncoder) addTime(final *ltsvEncoder, t time.Time) {
	if enc.timeFmt == "" {
		return
	}
	final.addKey(enc.timeLabel)
	final.bytes = t.AppendFormat(final.bytes, enc.timeFmt)
}

func (enc *ltsvEncoder) addMessage(final *ltsvEncoder, msg string) {
	final.addKey(enc.messageLabel)
	final.safeAddString(msg)
}

func (enc *ltsvEncoder) safeAddString(s string) {
	enc.bytes = append(enc.bytes, enc.replacer.Replace(s)...)
}

// safeAddString JSON-escapes a string and appends it to the internal buffer.
// Unlike the standard library's escaping function, it doesn't attempt to
// protect the user from browser vulnerabilities or JSONP-related problems.
func (enc *ltsvEncoder) safeAddJSONString(s string) {
	for i := 0; i < len(s); {
		if b := s[i]; b < utf8.RuneSelf {
			i++
			if 0x20 <= b && b != '\\' && b != '"' {
				enc.bytes = append(enc.bytes, b)
				continue
			}
			switch b {
			case '\\', '"':
				enc.bytes = append(enc.bytes, '\\', b)
			case '\n':
				enc.bytes = append(enc.bytes, '\\', 'n')
			case '\r':
				enc.bytes = append(enc.bytes, '\\', 'r')
			case '\t':
				enc.bytes = append(enc.bytes, '\\', 't')
			default:
				// Encode bytes < 0x20, except for the escape sequences above.
				enc.bytes = append(enc.bytes, `\u00`...)
				enc.bytes = append(enc.bytes, _hex[b>>4], _hex[b&0xF])
			}
			continue
		}
		c, size := utf8.DecodeRuneInString(s[i:])
		if c == utf8.RuneError && size == 1 {
			enc.bytes = append(enc.bytes, `\ufffd`...)
			i++
			continue
		}
		enc.bytes = append(enc.bytes, s[i:i+size]...)
		i += size
	}
}

// A LTSVOption is used to set options for a LTSV encoder.
type LTSVOption interface {
	apply(*ltsvEncoder)
}

type ltsvOptionFunc func(*ltsvEncoder)

func (opt ltsvOptionFunc) apply(enc *ltsvEncoder) {
	opt(enc)
}

// LTSVTimeFormat sets the format for log timestamps, using the same layout
// strings supported by time.Parse.
func LTSVTimeFormat(layout string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.timeFmt = layout
	})
}

// LTSVNoTime omits timestamps from the serialized log entries.
func LTSVNoTime() LTSVOption {
	return LTSVTimeFormat("")
}

// LTSVTimeLabel sets the label for log timestamps.
func LTSVTimeLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.timeLabel = label
	})
}

// LTSVLevelLabel sets the label for log levels.
func LTSVLevelLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.levelLabel = label
	})
}

// LTSVNoLevel omit levels from the serialized log entries.
func LTSVNoLevel() LTSVOption {
	return LTSVLevelLabel("")
}

// LTSVMessageLabel sets the label for log messages.
func LTSVMessageLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.messageLabel = label
	})
}

// LTSVReplacer sets the replacer for log values.
func LTSVReplacer(replacer Replacer) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.replacer = replacer
	})
}
<commit_msg>Add comment for setting time format and replacer<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
// Copyright (c) 2017 Hiraaki Nakamura
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package ltsv

import (
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/uber-go/zap"
)

const (
	// For JSON-escaping; see jsonEncoder.safeAddString below.
	_hex = "0123456789abcdef"
	// Initial buffer size for encoders.
	_initialBufSize = 1024
)

var (
	// errNilSink signals that Encoder.WriteEntry was called with a nil WriteSyncer.
	errNilSink = errors.New("can't write encoded message a nil WriteSyncer")
)

// Replacer is the interface that replaces a list of strings with replacements.
type Replacer interface {
	// Replace returns a copy of s with all replacements performed.
	Replace(s string) string
}

var ltsvPool = sync.Pool{New: func() interface{} {
	return &ltsvEncoder{
		bytes:        make([]byte, 0, _initialBufSize),
		timeLabel:    "time",
		levelLabel:   "level",
		messageLabel: "msg",
		replacer: strings.NewReplacer(
			"\t", "\\t",
			"\n", "\\n",
			"\r", "\\r",
		),
	}
}}

type ltsvEncoder struct {
	bytes        []byte
	timeLabel    string
	timeFmt      string
	levelLabel   string
	messageLabel string
	nestedLevel  int
	replacer     Replacer
}

// NewLTSVEncoder creates a line-oriented LTSV encoder.
// See http://ltsv.org/ for LTSV (Labeled Tab-separated Values).
// By default, the encoder uses RFC3339-formatted timestamps.
// You can change this format with the LTSVTimeFormat option.
//
// The tab \t, newline \n and carriage-return \r are escaped as
// \\t, \\n, \\r respectively.
// You can change this behavior with the LTSVReplacer option.
//
// Nested values are encoded in JSON format.  See Example (Nest).
func NewLTSVEncoder(options ...LTSVOption) zap.Encoder {
	enc := ltsvPool.Get().(*ltsvEncoder)
	enc.truncate()
	enc.timeFmt = time.RFC3339
	for _, opt := range options {
		opt.apply(enc)
	}
	return enc
}

func (enc *ltsvEncoder) Free() {
	ltsvPool.Put(enc)
}

func (enc *ltsvEncoder) AddString(key, val string) {
	enc.addKey(key)
	if enc.nestedLevel > 0 {
		enc.bytes = append(enc.bytes, '"')
		enc.safeAddJSONString(val)
		enc.bytes = append(enc.bytes, '"')
	} else {
		enc.safeAddString(val)
	}
}

func (enc *ltsvEncoder) AddBool(key string, val bool) {
	enc.addKey(key)
	enc.bytes = strconv.AppendBool(enc.bytes, val)
}

func (enc *ltsvEncoder) AddInt(key string, val int) {
	enc.AddInt64(key, int64(val))
}

func (enc *ltsvEncoder) AddInt64(key string, val int64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendInt(enc.bytes, val, 10)
}

func (enc *ltsvEncoder) AddUint(key string, val uint) {
	enc.AddUint64(key, uint64(val))
}

func (enc *ltsvEncoder) AddUint64(key string, val uint64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendUint(enc.bytes, val, 10)
}

func (enc *ltsvEncoder) AddUintptr(key string, val uintptr) {
	enc.addKey(key)
	enc.bytes = append(enc.bytes, "0x"...)
	enc.bytes = strconv.AppendUint(enc.bytes, uint64(val), 16)
}

func (enc *ltsvEncoder) AddFloat64(key string, val float64) {
	enc.addKey(key)
	enc.bytes = strconv.AppendFloat(enc.bytes, val, 'f', -1, 64)
}

func (enc *ltsvEncoder) AddMarshaler(key string, obj zap.LogMarshaler) error {
	enc.addKey(key)
	enc.nestedLevel++
	enc.bytes = append(enc.bytes, '{')
	err := obj.MarshalLog(enc)
	enc.bytes = append(enc.bytes, '}')
	enc.nestedLevel--
	return err
}

func (enc *ltsvEncoder) AddObject(key string, obj interface{}) error {
	enc.AddString(key, fmt.Sprintf("%+v", obj))
	return nil
}

func (enc *ltsvEncoder) Clone() zap.Encoder {
	clone := ltsvPool.Get().(*ltsvEncoder)
	clone.truncate()
	clone.bytes = append(clone.bytes, enc.bytes...)
	clone.timeLabel = enc.timeLabel
	clone.timeFmt = enc.timeFmt
	clone.levelLabel = enc.levelLabel
	clone.messageLabel = enc.messageLabel
	clone.nestedLevel = enc.nestedLevel
	clone.replacer = enc.replacer
	return clone
}

func (enc *ltsvEncoder) WriteEntry(sink io.Writer, msg string, lvl zap.Level, t time.Time) error {
	if sink == nil {
		return errNilSink
	}

	final := ltsvPool.Get().(*ltsvEncoder)
	final.truncate()
	enc.addTime(final, t)
	enc.addLevel(final, lvl)
	enc.addMessage(final, msg)

	if len(enc.bytes) > 0 {
		final.bytes = append(final.bytes, '\t')
		final.bytes = append(final.bytes, enc.bytes...)
	}
	final.bytes = append(final.bytes, '\n')

	expectedBytes := len(final.bytes)
	n, err := sink.Write(final.bytes)
	final.Free()
	if err != nil {
		return err
	}
	if n != expectedBytes {
		return fmt.Errorf("incomplete write: only wrote %v of %v bytes", n, expectedBytes)
	}
	return nil
}

func (enc *ltsvEncoder) truncate() {
	enc.bytes = enc.bytes[:0]
}

func (enc *ltsvEncoder) addKey(key string) {
	if enc.nestedLevel > 0 {
		last := len(enc.bytes) - 1
		// At some point, we'll also want to support arrays.
		if last >= 0 && enc.bytes[last] != '{' {
			enc.bytes = append(enc.bytes, ',')
		}
		enc.bytes = append(enc.bytes, '"')
		enc.safeAddJSONString(key)
		enc.bytes = append(enc.bytes, '"', ':')
	} else {
		if len(enc.bytes) > 0 {
			enc.bytes = append(enc.bytes, '\t')
		}
		enc.safeAddString(key)
		enc.bytes = append(enc.bytes, ':')
	}
}

func (enc *ltsvEncoder) addLevel(final *ltsvEncoder, lvl zap.Level) {
	final.addKey(enc.levelLabel)
	switch lvl {
	case zap.DebugLevel:
		final.bytes = append(final.bytes, 'D')
	case zap.InfoLevel:
		final.bytes = append(final.bytes, 'I')
	case zap.WarnLevel:
		final.bytes = append(final.bytes, 'W')
	case zap.ErrorLevel:
		final.bytes = append(final.bytes, 'E')
	case zap.PanicLevel:
		final.bytes = append(final.bytes, 'P')
	case zap.FatalLevel:
		final.bytes = append(final.bytes, 'F')
	default:
		final.bytes = strconv.AppendInt(final.bytes, int64(lvl), 10)
	}
}

func (enc *ltsvEncoder) addTime(final *ltsvEncoder, t time.Time) {
	if enc.timeFmt == "" {
		return
	}
	final.addKey(enc.timeLabel)
	final.bytes = t.AppendFormat(final.bytes, enc.timeFmt)
}

func (enc *ltsvEncoder) addMessage(final *ltsvEncoder, msg string) {
	final.addKey(enc.messageLabel)
	final.safeAddString(msg)
}

func (enc *ltsvEncoder) safeAddString(s string) {
	enc.bytes = append(enc.bytes, enc.replacer.Replace(s)...)
}

// safeAddString JSON-escapes a string and appends it to the internal buffer.
// Unlike the standard library's escaping function, it doesn't attempt to
// protect the user from browser vulnerabilities or JSONP-related problems.
func (enc *ltsvEncoder) safeAddJSONString(s string) {
	for i := 0; i < len(s); {
		if b := s[i]; b < utf8.RuneSelf {
			i++
			if 0x20 <= b && b != '\\' && b != '"' {
				enc.bytes = append(enc.bytes, b)
				continue
			}
			switch b {
			case '\\', '"':
				enc.bytes = append(enc.bytes, '\\', b)
			case '\n':
				enc.bytes = append(enc.bytes, '\\', 'n')
			case '\r':
				enc.bytes = append(enc.bytes, '\\', 'r')
			case '\t':
				enc.bytes = append(enc.bytes, '\\', 't')
			default:
				// Encode bytes < 0x20, except for the escape sequences above.
				enc.bytes = append(enc.bytes, `\u00`...)
				enc.bytes = append(enc.bytes, _hex[b>>4], _hex[b&0xF])
			}
			continue
		}
		c, size := utf8.DecodeRuneInString(s[i:])
		if c == utf8.RuneError && size == 1 {
			enc.bytes = append(enc.bytes, `\ufffd`...)
			i++
			continue
		}
		enc.bytes = append(enc.bytes, s[i:i+size]...)
		i += size
	}
}

// A LTSVOption is used to set options for a LTSV encoder.
type LTSVOption interface {
	apply(*ltsvEncoder)
}

type ltsvOptionFunc func(*ltsvEncoder)

func (opt ltsvOptionFunc) apply(enc *ltsvEncoder) {
	opt(enc)
}

// LTSVTimeFormat sets the format for log timestamps, using the same layout
// strings supported by time.Parse.
func LTSVTimeFormat(layout string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.timeFmt = layout
	})
}

// LTSVNoTime omits timestamps from the serialized log entries.
func LTSVNoTime() LTSVOption {
	return LTSVTimeFormat("")
}

// LTSVTimeLabel sets the label for log timestamps.
func LTSVTimeLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.timeLabel = label
	})
}

// LTSVLevelLabel sets the label for log levels.
func LTSVLevelLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.levelLabel = label
	})
}

// LTSVNoLevel omit levels from the serialized log entries.
func LTSVNoLevel() LTSVOption {
	return LTSVLevelLabel("")
}

// LTSVMessageLabel sets the label for log messages.
func LTSVMessageLabel(label string) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.messageLabel = label
	})
}

// LTSVReplacer sets the replacer for log values.
func LTSVReplacer(replacer Replacer) LTSVOption {
	return ltsvOptionFunc(func(enc *ltsvEncoder) {
		enc.replacer = replacer
	})
}
<|endoftext|>