<commit_before>package game

import (
	"bufio"
	"encoding/gob"
	"io"
	"log"

	"github.com/ghthor/aodd/game/datastore"
	"github.com/ghthor/engine/rpg2d"
	"github.com/ghthor/engine/rpg2d/entity"
	"golang.org/x/net/websocket"
)

// Used to determine the next type that's in the
// buffer so we can decode it into a real value.
// We'll decode an encoded type and switch on its
// value so we'll have the correct value to decode
// into.
type EncodedType int

//go:generate stringer -type=EncodedType
const (
	ET_ERROR EncodedType = iota
	ET_DISCONNECT

	ET_REQ_LOGIN
	ET_REQ_CREATE

	ET_RESP_AUTH_FAILED
	ET_RESP_ACTOR_EXISTS
	ET_RESP_ACTOR_DOESNT_EXIST

	ET_RESP_LOGIN_SUCCESS
	ET_RESP_CREATE_SUCCESS

	ET_WORLD_STATE
	ET_WORLD_STATE_DIFF

	ET_REQ_MOVE
	ET_REQ_USE
	ER_REQ_CHAT
)

type ReqLogin struct{ Name, Password string }
type ReqCreate struct{ Name, Password string }

type RespAuthFailed struct{ Name string }
type RespActorExists struct{ Name string }
type RespActorDoesntExist struct{ Name, Password string }

const RespDisconnect = "disconnected"

func init() {
	// Pre login Request/Response types
	gob.Register(ReqLogin{})
	gob.Register(ReqCreate{})

	gob.Register(RespAuthFailed{})
	gob.Register(RespActorExists{})
	gob.Register(RespActorDoesntExist{})

	// ActorEntityState used for login/create success
	gob.Register(ActorEntityState{})

	// Engine types
	gob.Register(rpg2d.WorldState{})
	gob.Register(rpg2d.WorldStateDiff{})
	gob.Register(rpg2d.TerrainMapState{})

	// Other entity states
	gob.Register(SayEntityState{})
	gob.Register(AssailEntityState{})

	// Cmd Requests. They have no responses.
	gob.Register(MoveRequest{})
	gob.Register(UseRequest{})
	gob.Register(ChatRequest{})
}

type GobConn interface {
	EncodeAndSend(EncodedType, interface{}) error
	ReadNextType() (EncodedType, error)
	Decode(interface{}) error
}

type gobConn struct {
	enc  *gob.Encoder
	wbuf *bufio.Writer

	*gob.Decoder
}

func (c gobConn) EncodeAndSend(t EncodedType, ev interface{}) error {
	err := c.enc.Encode(t)
	if err != nil {
		return err
	}

	err = c.enc.Encode(ev)
	if err != nil {
		return err
	}

	return c.wbuf.Flush()
}

func (c gobConn) ReadNextType() (t EncodedType, err error) {
	err = c.Decoder.Decode(&t)
	return
}

func NewGobConn(rw io.ReadWriter) GobConn {
	wbuf := bufio.NewWriter(rw)
	enc := gob.NewEncoder(wbuf)
	dec := gob.NewDecoder(rw)

	return gobConn{
		enc:  enc,
		wbuf: wbuf,

		Decoder: dec,
	}
}

type InputReceiver interface {
	SubmitMoveRequest(MoveRequest)
	SubmitUseRequest(UseRequest)
	SubmitChatRequest(ChatRequest)

	Close()
}

type serverConn struct {
	GobConn

	datastore datastore.Datastore

	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)
	actor    InputReceiver
}

type ActorConn interface {
	Run() error
}

type stateFn func() (stateFn, error)

func (c *serverConn) handleLogin() (stateFn, error) {
	eType, err := c.ReadNextType()
	if err != nil {
		return nil, err
	}

	switch eType {
	case ET_REQ_LOGIN:
		return c.handleLoginReq, nil
	case ET_REQ_CREATE:
		return c.handleCreateReq, nil

	default:
		log.Println("unexpected encoded type: ", eType)
	}

	return c.handleLogin, nil
}

func (c *serverConn) handleLoginReq() (stateFn, error) {
	var r ReqLogin
	err := c.Decode(&r)
	if err != nil {
		return nil, err
	}

	actor, exists := c.datastore.ActorExists(r.Name)
	if !exists {
		err := c.EncodeAndSend(ET_RESP_ACTOR_DOESNT_EXIST, RespActorDoesntExist{
			r.Name, r.Password,
		})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	if !actor.Authenticate(r.Name, r.Password) {
		err := c.EncodeAndSend(ET_RESP_AUTH_FAILED, RespAuthFailed{r.Name})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	err = c.EncodeAndSend(ET_RESP_LOGIN_SUCCESS, c.login(actor))
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) handleCreateReq() (stateFn, error) {
	var r ReqCreate
	err := c.Decode(&r)
	if err != nil {
		return nil, err
	}

	_, exists := c.datastore.ActorExists(r.Name)
	if exists {
		err := c.EncodeAndSend(ET_RESP_ACTOR_EXISTS, RespActorExists{r.Name})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	actor, err := c.datastore.AddActor(r.Name, r.Password)
	if err != nil {
		// TODO Instead of terminating the connection here
		//      we should retry contacting the database a
		//      few times
		return nil, err
	}

	err = c.EncodeAndSend(ET_RESP_CREATE_SUCCESS, c.login(actor))
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) handleInputReq() (stateFn, error) {
	_, err := c.ReadNextType()
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) login(dsactor datastore.Actor) (state entity.State) {
	c.actor, state = c.newActor(dsactor, c)
	return state
}

func (c serverConn) Run() (err error) {
	f := c.handleLogin
	for f != nil && err == nil {
		f, err = f()
	}

	if c.actor != nil {
		c.actor.Close()
	}

	return
}

func (c serverConn) WriteWorldState(s rpg2d.WorldState) error {
	return c.EncodeAndSend(ET_WORLD_STATE, s)
}

func (c serverConn) WriteWorldStateDiff(s rpg2d.WorldStateDiff) error {
	return c.EncodeAndSend(ET_WORLD_STATE_DIFF, s)
}

func NewActorGobConn(
	rw io.ReadWriter,
	ds datastore.Datastore,
	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)) ActorConn {
	return serverConn{
		GobConn:   NewGobConn(rw),
		datastore: ds,
		newActor:  newActor,
	}
}

func newGobWebsocketHandler(
	ds datastore.Datastore,
	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)) websocket.Handler {
	return func(ws *websocket.Conn) {
		ws.PayloadType = websocket.BinaryFrame

		c := NewActorGobConn(ws, ds, newActor)

		// Blocks until the connection is disconnected
		err := c.Run()

		if err != nil {
			log.Printf("packet handler terminated: %v", err)
		}
	}
}
<commit_msg>[game] fix spelling error in constant<commit_after>package game

import (
	"bufio"
	"encoding/gob"
	"io"
	"log"

	"github.com/ghthor/aodd/game/datastore"
	"github.com/ghthor/engine/rpg2d"
	"github.com/ghthor/engine/rpg2d/entity"
	"golang.org/x/net/websocket"
)

// Used to determine the next type that's in the
// buffer so we can decode it into a real value.
// We'll decode an encoded type and switch on its
// value so we'll have the correct value to decode
// into.
type EncodedType int

//go:generate stringer -type=EncodedType
const (
	ET_ERROR EncodedType = iota
	ET_DISCONNECT

	ET_REQ_LOGIN
	ET_REQ_CREATE

	ET_RESP_AUTH_FAILED
	ET_RESP_ACTOR_EXISTS
	ET_RESP_ACTOR_DOESNT_EXIST

	ET_RESP_LOGIN_SUCCESS
	ET_RESP_CREATE_SUCCESS

	ET_WORLD_STATE
	ET_WORLD_STATE_DIFF

	ET_REQ_MOVE
	ET_REQ_USE
	ET_REQ_CHAT
)

type ReqLogin struct{ Name, Password string }
type ReqCreate struct{ Name, Password string }

type RespAuthFailed struct{ Name string }
type RespActorExists struct{ Name string }
type RespActorDoesntExist struct{ Name, Password string }

const RespDisconnect = "disconnected"

func init() {
	// Pre login Request/Response types
	gob.Register(ReqLogin{})
	gob.Register(ReqCreate{})

	gob.Register(RespAuthFailed{})
	gob.Register(RespActorExists{})
	gob.Register(RespActorDoesntExist{})

	// ActorEntityState used for login/create success
	gob.Register(ActorEntityState{})

	// Engine types
	gob.Register(rpg2d.WorldState{})
	gob.Register(rpg2d.WorldStateDiff{})
	gob.Register(rpg2d.TerrainMapState{})

	// Other entity states
	gob.Register(SayEntityState{})
	gob.Register(AssailEntityState{})

	// Cmd Requests. They have no responses.
	gob.Register(MoveRequest{})
	gob.Register(UseRequest{})
	gob.Register(ChatRequest{})
}

type GobConn interface {
	EncodeAndSend(EncodedType, interface{}) error
	ReadNextType() (EncodedType, error)
	Decode(interface{}) error
}

type gobConn struct {
	enc  *gob.Encoder
	wbuf *bufio.Writer

	*gob.Decoder
}

func (c gobConn) EncodeAndSend(t EncodedType, ev interface{}) error {
	err := c.enc.Encode(t)
	if err != nil {
		return err
	}

	err = c.enc.Encode(ev)
	if err != nil {
		return err
	}

	return c.wbuf.Flush()
}

func (c gobConn) ReadNextType() (t EncodedType, err error) {
	err = c.Decoder.Decode(&t)
	return
}

func NewGobConn(rw io.ReadWriter) GobConn {
	wbuf := bufio.NewWriter(rw)
	enc := gob.NewEncoder(wbuf)
	dec := gob.NewDecoder(rw)

	return gobConn{
		enc:  enc,
		wbuf: wbuf,

		Decoder: dec,
	}
}

type InputReceiver interface {
	SubmitMoveRequest(MoveRequest)
	SubmitUseRequest(UseRequest)
	SubmitChatRequest(ChatRequest)

	Close()
}

type serverConn struct {
	GobConn

	datastore datastore.Datastore

	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)
	actor    InputReceiver
}

type ActorConn interface {
	Run() error
}

type stateFn func() (stateFn, error)

func (c *serverConn) handleLogin() (stateFn, error) {
	eType, err := c.ReadNextType()
	if err != nil {
		return nil, err
	}

	switch eType {
	case ET_REQ_LOGIN:
		return c.handleLoginReq, nil
	case ET_REQ_CREATE:
		return c.handleCreateReq, nil

	default:
		log.Println("unexpected encoded type: ", eType)
	}

	return c.handleLogin, nil
}

func (c *serverConn) handleLoginReq() (stateFn, error) {
	var r ReqLogin
	err := c.Decode(&r)
	if err != nil {
		return nil, err
	}

	actor, exists := c.datastore.ActorExists(r.Name)
	if !exists {
		err := c.EncodeAndSend(ET_RESP_ACTOR_DOESNT_EXIST, RespActorDoesntExist{
			r.Name, r.Password,
		})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	if !actor.Authenticate(r.Name, r.Password) {
		err := c.EncodeAndSend(ET_RESP_AUTH_FAILED, RespAuthFailed{r.Name})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	err = c.EncodeAndSend(ET_RESP_LOGIN_SUCCESS, c.login(actor))
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) handleCreateReq() (stateFn, error) {
	var r ReqCreate
	err := c.Decode(&r)
	if err != nil {
		return nil, err
	}

	_, exists := c.datastore.ActorExists(r.Name)
	if exists {
		err := c.EncodeAndSend(ET_RESP_ACTOR_EXISTS, RespActorExists{r.Name})
		if err != nil {
			return nil, err
		}

		return c.handleLogin, nil
	}

	actor, err := c.datastore.AddActor(r.Name, r.Password)
	if err != nil {
		// TODO Instead of terminating the connection here
		//      we should retry contacting the database a
		//      few times
		return nil, err
	}

	err = c.EncodeAndSend(ET_RESP_CREATE_SUCCESS, c.login(actor))
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) handleInputReq() (stateFn, error) {
	_, err := c.ReadNextType()
	if err != nil {
		return nil, err
	}

	return c.handleInputReq, nil
}

func (c *serverConn) login(dsactor datastore.Actor) (state entity.State) {
	c.actor, state = c.newActor(dsactor, c)
	return state
}

func (c serverConn) Run() (err error) {
	f := c.handleLogin
	for f != nil && err == nil {
		f, err = f()
	}

	if c.actor != nil {
		c.actor.Close()
	}

	return
}

func (c serverConn) WriteWorldState(s rpg2d.WorldState) error {
	return c.EncodeAndSend(ET_WORLD_STATE, s)
}

func (c serverConn) WriteWorldStateDiff(s rpg2d.WorldStateDiff) error {
	return c.EncodeAndSend(ET_WORLD_STATE_DIFF, s)
}

func NewActorGobConn(
	rw io.ReadWriter,
	ds datastore.Datastore,
	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)) ActorConn {
	return serverConn{
		GobConn:   NewGobConn(rw),
		datastore: ds,
		newActor:  newActor,
	}
}

func newGobWebsocketHandler(
	ds datastore.Datastore,
	newActor func(datastore.Actor, StateWriter) (InputReceiver, entity.State)) websocket.Handler {
	return func(ws *websocket.Conn) {
		ws.PayloadType = websocket.BinaryFrame

		c := NewActorGobConn(ws, ds, newActor)

		// Blocks until the connection is disconnected
		err := c.Run()

		if err != nil {
			log.Printf("packet handler terminated: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package diff

import (
	"encoding/json"
	"fmt"

	"github.com/fatih/color"
)

// Output is used to configure the output of the Strings and StringIndent functions.
type Output struct {
	Indent     string
	ShowTypes  bool
	Colorized  bool
	JSON       bool
	JSONValues bool
}

func (o Output) red(v interface{}) string {
	var s string

	switch {
	default:
		s = fmt.Sprintf("%v", v)
	case o.ShowTypes:
		s = fmt.Sprintf("%T %v", v, v)
	case o.JSONValues:
		s = jsonString(v)
	}

	if !o.Colorized {
		return s
	}

	return color.RedString("%s", s)
}

func (o Output) green(v interface{}) string {
	var s string

	switch {
	default:
		s = fmt.Sprintf("%v", v)
	case o.ShowTypes:
		s = fmt.Sprintf("%T %v", v, v)
	case o.JSONValues:
		s = jsonString(v)
	}

	if !o.Colorized {
		return s
	}

	return color.GreenString("%s", s)
}

func (o Output) white(v interface{}) string {
	var s string

	switch {
	default:
		s = fmt.Sprintf("%v", v)
	case o.ShowTypes:
		s = fmt.Sprintf("%T %v", v, v)
	case o.JSONValues:
		s = jsonString(v)
	}

	return s
}

func (o Output) typ(v interface{}) string {
	if o.ShowTypes {
		return fmt.Sprintf("%T ", v)
	}

	return ""
}

func newLineSeparatorString(conf Output) string {
	if conf.JSON {
		return ",\n"
	}

	return "\n"
}

func jsonString(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic(fmt.Errorf("unexpected error marshaling value: %s", err))
	}

	return string(b)
}
<commit_msg>deduplicating code in diff/output.go<commit_after>package diff

import (
	"encoding/json"
	"fmt"

	"github.com/fatih/color"
)

// Output is used to configure the output of the Strings and StringIndent functions.
type Output struct {
	Indent     string
	ShowTypes  bool
	Colorized  bool
	JSON       bool
	JSONValues bool
}

type colorFn func(format string, a ...interface{}) string

var whiteFn colorFn = nil

func (o Output) red(v interface{}) string {
	return o.applyColor(v, color.RedString)
}

func (o Output) green(v interface{}) string {
	return o.applyColor(v, color.GreenString)
}

func (o Output) white(v interface{}) string {
	return o.applyColor(v, whiteFn)
}

func (o Output) applyColor(v interface{}, fn colorFn) string {
	var s string

	switch {
	default:
		s = fmt.Sprintf("%v", v)
	case o.ShowTypes:
		s = fmt.Sprintf("%T %v", v, v)
	case o.JSONValues:
		s = jsonString(v)
	}

	if !o.Colorized || fn == nil {
		return s
	}

	return fn("%s", s)
}

func (o Output) typ(v interface{}) string {
	if o.ShowTypes {
		return fmt.Sprintf("%T ", v)
	}

	return ""
}

func newLineSeparatorString(conf Output) string {
	if conf.JSON {
		return ",\n"
	}

	return "\n"
}

func jsonString(v interface{}) string {
	b, err := json.Marshal(v)
	if err != nil {
		panic(fmt.Errorf("unexpected error marshaling value: %s", err))
	}

	return string(b)
}
<|endoftext|>


<commit_before>package manifest

import (
	"encoding/json"

	"github.com/docker/distribution/digest"
	"github.com/docker/libtrust"
)

// Versioned provides a struct with just the manifest schemaVersion. Incoming
// content with unknown schema version can be decoded against this struct to
// check the version.
type Versioned struct {
	// SchemaVersion is the image manifest schema that this image follows
	SchemaVersion int `json:"schemaVersion"`
}

// Manifest provides the base accessible fields for working with V2 image
// format in the registry.
type Manifest struct {
	Versioned

	// Name is the name of the image's repository
	Name string `json:"name"`

	// Tag is the tag of the image specified by this manifest
	Tag string `json:"tag"`

	// Architecture is the host architecture on which this image is intended to
	// run
	Architecture string `json:"architecture"`

	// FSLayers is a list of filesystem layer blobSums contained in this image
	FSLayers []FSLayer `json:"fsLayers"`

	// History is a list of unstructured historical data for v1 compatibility
	History []History `json:"history"`
}

// SignedManifest provides an envelope for a signed image manifest, including
// the format sensitive raw bytes. It contains fields to
type SignedManifest struct {
	Manifest

	// Raw is the byte representation of the ImageManifest, used for signature
	// verification. The value of Raw must be used directly during
	// serialization, or the signature check will fail. The manifest byte
	// representation cannot change or it will have to be re-signed.
	Raw []byte `json:"-"`
}

// UnmarshalJSON populates a new ImageManifest struct from JSON data.
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
	var manifest Manifest
	if err := json.Unmarshal(b, &manifest); err != nil {
		return err
	}

	sm.Manifest = manifest
	sm.Raw = make([]byte, len(b), len(b))
	copy(sm.Raw, b)

	return nil
}

// Payload returns the raw, signed content of the signed manifest. The
// contents can be used to calculate the content identifier.
func (sm *SignedManifest) Payload() ([]byte, error) {
	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
	if err != nil {
		return nil, err
	}

	// Resolve the payload in the manifest.
	return jsig.Payload()
}

// Signatures returns the signatures as provided by
// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
// signatures.
func (sm *SignedManifest) Signatures() ([][]byte, error) {
	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
	if err != nil {
		return nil, err
	}

	// Resolve the payload in the manifest.
	return jsig.Signatures()
}

// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner
// contents. Applications requiring a marshaled signed manifest should simply
// use Raw directly, since the the content produced by json.Marshal will be
// compacted and will fail signature checks.
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
	if len(sm.Raw) > 0 {
		return sm.Raw, nil
	}

	// If the raw data is not available, just dump the inner content.
	return json.Marshal(&sm.Manifest)
}

// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
	// BlobSum is the tarsum of the referenced filesystem image layer
	BlobSum digest.Digest `json:"blobSum"`
}

// History stores unstructured v1 compatibility information
type History struct {
	// V1Compatibility is the raw v1 compatibility information
	V1Compatibility string `json:"v1Compatibility"`
}
<commit_msg>Specify manifest schema version 1 media type<commit_after>package manifest

import (
	"encoding/json"

	"github.com/docker/distribution/digest"
	"github.com/docker/libtrust"
)

// TODO(stevvooe): When we rev the manifest format, the contents of this
// package should me moved to manifest/v1.

const (
	// ManifestMediaType specifies the mediaType for the current version. Note
	// that for schema version 1, the the media is optionally
	// "application/json".
	ManifestMediaType = "application/vnd.docker.distribution.manifest.v1+json"
)

// Versioned provides a struct with just the manifest schemaVersion. Incoming
// content with unknown schema version can be decoded against this struct to
// check the version.
type Versioned struct {
	// SchemaVersion is the image manifest schema that this image follows
	SchemaVersion int `json:"schemaVersion"`
}

// Manifest provides the base accessible fields for working with V2 image
// format in the registry.
type Manifest struct {
	Versioned

	// Name is the name of the image's repository
	Name string `json:"name"`

	// Tag is the tag of the image specified by this manifest
	Tag string `json:"tag"`

	// Architecture is the host architecture on which this image is intended to
	// run
	Architecture string `json:"architecture"`

	// FSLayers is a list of filesystem layer blobSums contained in this image
	FSLayers []FSLayer `json:"fsLayers"`

	// History is a list of unstructured historical data for v1 compatibility
	History []History `json:"history"`
}

// SignedManifest provides an envelope for a signed image manifest, including
// the format sensitive raw bytes. It contains fields to
type SignedManifest struct {
	Manifest

	// Raw is the byte representation of the ImageManifest, used for signature
	// verification. The value of Raw must be used directly during
	// serialization, or the signature check will fail. The manifest byte
	// representation cannot change or it will have to be re-signed.
	Raw []byte `json:"-"`
}

// UnmarshalJSON populates a new ImageManifest struct from JSON data.
func (sm *SignedManifest) UnmarshalJSON(b []byte) error {
	var manifest Manifest
	if err := json.Unmarshal(b, &manifest); err != nil {
		return err
	}

	sm.Manifest = manifest
	sm.Raw = make([]byte, len(b), len(b))
	copy(sm.Raw, b)

	return nil
}

// Payload returns the raw, signed content of the signed manifest. The
// contents can be used to calculate the content identifier.
func (sm *SignedManifest) Payload() ([]byte, error) {
	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
	if err != nil {
		return nil, err
	}

	// Resolve the payload in the manifest.
	return jsig.Payload()
}

// Signatures returns the signatures as provided by
// (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws
// signatures.
func (sm *SignedManifest) Signatures() ([][]byte, error) {
	jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures")
	if err != nil {
		return nil, err
	}

	// Resolve the payload in the manifest.
	return jsig.Signatures()
}

// MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner
// contents. Applications requiring a marshaled signed manifest should simply
// use Raw directly, since the the content produced by json.Marshal will be
// compacted and will fail signature checks.
func (sm *SignedManifest) MarshalJSON() ([]byte, error) {
	if len(sm.Raw) > 0 {
		return sm.Raw, nil
	}

	// If the raw data is not available, just dump the inner content.
	return json.Marshal(&sm.Manifest)
}

// FSLayer is a container struct for BlobSums defined in an image manifest
type FSLayer struct {
	// BlobSum is the tarsum of the referenced filesystem image layer
	BlobSum digest.Digest `json:"blobSum"`
}

// History stores unstructured v1 compatibility information
type History struct {
	// V1Compatibility is the raw v1 compatibility information
	V1Compatibility string `json:"v1Compatibility"`
}
<|endoftext|>


<commit_before>package sender

import (
	"io"

	"github.com/chrislusf/netchan/service_discovery/client"
	"github.com/chrislusf/netchan/util"
)

func NewChannel(name string, leader string) (chan []byte, error) {
	ss := NewSenderServer()
	ss.Init()

	ch := make(chan []byte, 1000)
	ss.Handler = func(in io.Reader, out io.WriteCloser) {
		buf := make([]byte, 4)
		for data := range ch {
			util.Uint32toBytes(buf, uint32(len(data)))
			out.Write(buf)
			out.Write(data)
		}
	}

	go ss.Loop()

	b := client.NewHeartBeater(name, ss.Port, leader)
	go b.Start()

	return ch, nil
}
<commit_msg>switch to disk based queue<commit_after>package sender

import (
	"io"
	"os"
	"time"

	"github.com/chrislusf/netchan/service_discovery/client"
	"github.com/chrislusf/netchan/util"
	"github.com/chrislusf/netchan/queue"
)

func NewChannel(name string, leader string) (chan []byte, error) {
	ss := NewSenderServer()
	ss.Init()

	dq := queue.NewDiskQueue(name, os.TempDir(), 1024*1024, 2500, 2*time.Second)
	ss.Handler = func(in io.Reader, out io.WriteCloser) {
		buf := make([]byte, 4)
		for data := range dq.ReadChan() {
			util.Uint32toBytes(buf, uint32(len(data)))
			out.Write(buf)
			out.Write(data)
		}
	}

	ch := make(chan []byte, 1000)
	go func(){
		for data := range ch {
			dq.Put(data)
		}
	}()

	go ss.Loop()

	b := client.NewHeartBeater(name, ss.Port, leader)
	go b.Start()

	return ch, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/mikespook/golib/log"
	"github.com/mikespook/golib/signal"
	"github.com/mikespook/possum"
	"gopkg.in/yaml.v2"
)

type configLog struct {
	File, Level string
}

type Config struct {
	Addr  string
	PProf string
	Log   configLog
	Test  bool
}

func LoadConfig(filename string) (config *Config, err error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	err = yaml.Unmarshal(data, &config)
	return
}

var configFile string

func init() {
	flag.StringVar(&configFile, "config", "", "Path to the configuration file")
	flag.Parse()
}

func main() {
	if configFile == "" {
		flag.Usage()
		return
	}
	config, err := LoadConfig(configFile)
	if err != nil {
		log.Error(err)
		flag.Usage()
		return
	}
	if err := log.Init(config.Log.File, log.StrToLevel(config.Log.Level)); err != nil {
		log.Error(err)
	}

	if config.Test {
		if err := possum.InitViewWatcher("*.html", possum.InitHtmlTemplates, nil); err != nil {
			log.Error(err)
			return
		}
		if err := possum.InitViewWatcher("*.html", possum.InitTextTemplates, nil); err != nil {
			log.Error(err)
			return
		}
	} else {
		if err := possum.InitHtmlTemplates("*.html"); err != nil {
			log.Error(err)
			return
		}

		if err := possum.InitTextTemplates("*.html"); err != nil {
			log.Error(err)
			return
		}
	}

	mux := possum.NewServerMux()
	mux.HandleFunc("/json", helloworld, possum.JsonView{})
	mux.HandleFunc("/html", helloworld, possum.NewHtmlView("base.html", "utf-8"))
	mux.HandleFunc("/text", helloworld, possum.NewTextView("base.html", "utf-8"))
	mux.HandleFunc("/project.css", nil, possum.NewStaticFileView("project.css", "text/css"))
	mux.HandleFunc("/img.jpg", nil, possum.NewStaticFileView("img.jpg", "image/jpeg"))

	if config.PProf != "" {
		log.Messagef("PProf: http://%s%s", config.Addr, config.PProf)
		mux.InitPProf(config.PProf)
	}
	log.Messagef("Addr: %s", config.Addr)
	go http.ListenAndServe(config.Addr, mux)
	sh := signal.NewHandler()
	sh.Bind(os.Interrupt, func() bool {
		log.Message("Exit")
		return true
	})
	sh.Loop()
}

func css(ctx *possum.Context) error {
	return nil
}

func helloworld(ctx *possum.Context) error {
	ctx.Response.Status = http.StatusCreated
	ctx.Response.Data = map[string]interface{}{
		"content": map[string]string{
			"msg":    "hello",
			"target": "world",
		},
	}
	ctx.Response.Header().Set("Test", "Hello world")
	return nil
}
<commit_msg>upgrade for new log api<commit_after>package main

import (
	"flag"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/mikespook/golib/log"
	"github.com/mikespook/golib/signal"
	"github.com/mikespook/possum"
	"gopkg.in/yaml.v2"
)

type configLog struct {
	File, Level string
}

type Config struct {
	Addr  string
	PProf string
	Log   configLog
	Test  bool
}

func LoadConfig(filename string) (config *Config, err error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	err = yaml.Unmarshal(data, &config)
	return
}

var configFile string

func init() {
	flag.StringVar(&configFile, "config", "", "Path to the configuration file")
	flag.Parse()
}

func main() {
	if configFile == "" {
		flag.Usage()
		return
	}
	config, err := LoadConfig(configFile)
	if err != nil {
		log.Error(err)
		flag.Usage()
		return
	}
	if err := log.Init(config.Log.File, log.StrToLevel(config.Log.Level), log.DefaultCallDepth); err != nil {
		log.Error(err)
	}

	if config.Test {
		if err := possum.InitViewWatcher("*.html", possum.InitHtmlTemplates, nil); err != nil {
			log.Error(err)
			return
		}
		if err := possum.InitViewWatcher("*.html", possum.InitTextTemplates, nil); err != nil {
			log.Error(err)
			return
		}
	} else {
		if err := possum.InitHtmlTemplates("*.html"); err != nil {
			log.Error(err)
			return
		}

		if err := possum.InitTextTemplates("*.html"); err != nil {
			log.Error(err)
			return
		}
	}

	mux := possum.NewServerMux()
	mux.HandleFunc("/json", helloworld, possum.JsonView{})
	mux.HandleFunc("/html", helloworld, possum.NewHtmlView("base.html", "utf-8"))
	mux.HandleFunc("/text", helloworld, possum.NewTextView("base.html", "utf-8"))
	mux.HandleFunc("/project.css", nil, possum.NewStaticFileView("project.css", "text/css"))
	mux.HandleFunc("/img.jpg", nil, possum.NewStaticFileView("img.jpg", "image/jpeg"))

	if config.PProf != "" {
		log.Messagef("PProf: http://%s%s", config.Addr, config.PProf)
		mux.InitPProf(config.PProf)
	}
	log.Messagef("Addr: %s", config.Addr)
	go http.ListenAndServe(config.Addr, mux)
	sh := signal.NewHandler()
	sh.Bind(os.Interrupt, func() bool {
		log.Message("Exit")
		return true
	})
	sh.Loop()
}

func css(ctx *possum.Context) error {
	return nil
}

func helloworld(ctx *possum.Context) error {
	ctx.Response.Status = http.StatusCreated
	ctx.Response.Data = map[string]interface{}{
		"content": map[string]string{
			"msg":    "hello",
			"target": "world",
		},
	}
	ctx.Response.Header().Set("Test", "Hello world")
	return nil
}
<|endoftext|>


<commit_before>package demo_test

import (
	"fmt"
	"strings"
	"testing"

	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
	"github.com/stretchr/testify/assert"
)

type Country struct {
	Code string
	Name string
}

type City struct {
	Code        string
	Name        string
	CountryCode string `db:"country_code"`
}

func Test(t *testing.T) {
	db, err := sqlx.Open("sqlite3", ":memory:")
	assert.Nil(t, err)
	assert.NotNil(t, db)

	err = db.Ping()
	assert.Nil(t, err)

	_, err = db.Exec(`
	CREATE TABLE IF NOT EXISTS Country (
		code  TEXT  PRIMARY KEY  NOT NULL,
		name  TEXT
	);
	CREATE TABLE IF NOT EXISTS City (
		code          TEXT  PRIMARY KEY  NOT NULL,
		name          TEXT,
		country_code  TEXT
	);`)
	assert.Nil(t, err)

	tx := db.MustBegin()
	tx.MustExec("INSERT INTO Country (code, name) VALUES (?, ?)", "PT", "Portugal")
	tx.MustExec("INSERT INTO Country (code, name) VALUES (?, ?)", "ES", "Spain")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "OPO", "Porto", "PT")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "LIS", "Lisbon", "PT")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "BAR", "Barcelona", "SP")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "MAD", "Madrid", "SP")
	err = tx.Commit()
	assert.Nil(t, err)

	rows, err := db.Queryx("SELECT code, name FROM Country")
	assert.Nil(t, err)
	for rows.Next() {
		country := Country{}
		err = rows.StructScan(&country)
		assert.Nil(t, err)
	}

	country := Country{}
	err = db.Get(&country, "SELECT * FROM Country WHERE code = ?", "PT")
	assert.Nil(t, err)
	assert.Equal(t, Country{Code: "PT", Name: "Portugal"}, country)

	cities := []City{}
	err = db.Select(&cities, "SELECT * FROM City")
	assert.Nil(t, err)
	assert.Equal(t, 4, len(cities))

	tx = db.MustBegin()
	tx.MustExec("UPDATE City SET name = ? WHERE code = ?", "Lisboa", "LIS")
	err = tx.Commit()
	assert.Nil(t, err)

	lisbon := City{}
	err = db.Get(&lisbon, "SELECT name FROM City WHERE code = ?", "LIS")
	assert.Nil(t, err)
	assert.Equal(t, "Lisboa", lisbon.Name)
	assert.Equal(t, "", lisbon.Code) // because code column is not part of the select statement

	tx = db.MustBegin()
	tx.MustExec("DELETE FROM City WHERE country_code = ?", "SP")
	err = tx.Commit()
	assert.Nil(t, err)

	cities = []City{}
	err = db.Select(&cities, "SELECT * FROM City WHERE country_code = ?", "SP")
	assert.Nil(t, err)
	assert.Equal(t, 0, len(cities))

	cities = []City{}
	questionMarks, cityCodes := createArrays([]string{"OPO", "LIS"})
	err = db.Select(&cities, fmt.Sprintf("SELECT * FROM City WHERE code in (%s)", questionMarks), cityCodes...)
	assert.Nil(t, err)
	assert.Equal(t, 2, len(cities))

	paris := City{}
	err = db.Get(&paris, "SELECT * FROM City WHERE code = ?", "PAR")
	assert.NotNil(t, err)
	assert.Equal(t, "", paris.Name)
}

func createArrays(values []string) (string, []interface{}) {
	var questionMarks []string
	var valueMarks []interface{}
	for _, value := range values {
		questionMarks = append(questionMarks, "?")
		valueMarks = append(valueMarks, value)
	}
	return strings.Join(questionMarks, ","), valueMarks
}
<commit_msg>update<commit_after>package demo_test

import (
	"fmt"
	"strings"
	"testing"

	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
	"github.com/stretchr/testify/assert"
)

type Country struct {
	Code string
	Name string
}

type City struct {
	Code        string
	Name        string
	CountryCode string `db:"country_code"`
}

func Test(t *testing.T) {
	db, err := sqlx.Open("sqlite3", ":memory:")
	assert.Nil(t, err)
	assert.NotNil(t, db)

	err = db.Ping()
	assert.Nil(t, err)

	_, err = db.Exec(`
	CREATE TABLE IF NOT EXISTS Country (
		code  TEXT  PRIMARY KEY  NOT NULL,
		name  TEXT
	);
	CREATE TABLE IF NOT EXISTS City (
		code          TEXT  PRIMARY KEY  NOT NULL,
		name          TEXT,
		country_code  TEXT
	);`)
	assert.Nil(t, err)

	tx := db.MustBegin()
	tx.MustExec("INSERT INTO Country (code, name) VALUES (?, ?)", "PT", "Portugal")
	tx.MustExec("INSERT INTO Country (code, name) VALUES (?, ?)", "ES", "Spain")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "OPO", "Porto", "PT")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "LIS", "Lisbon", "PT")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "BAR", "Barcelona", "SP")
	tx.MustExec("INSERT INTO City (code, name, country_code) VALUES (?, ?, ?)", "MAD", "Madrid", "SP")
	err = tx.Commit()
	assert.Nil(t, err)

	rows, err := db.Queryx("SELECT code, name FROM Country")
	assert.Nil(t, err)
	for rows.Next() {
		country := Country{}
		err = rows.StructScan(&country)
		assert.Nil(t, err)
	}

	country := Country{}
	err = db.Get(&country, "SELECT * FROM Country WHERE code = ?", "PT")
	assert.Nil(t, err)
	assert.Equal(t, Country{Code: "PT", Name: "Portugal"}, country)

	cities := []City{}
	err = db.Select(&cities, "SELECT * FROM City")
	assert.Nil(t, err)
	assert.Equal(t, 4, len(cities))

	tx = db.MustBegin()
	tx.MustExec("UPDATE City SET name = ? WHERE code = ?", "Lisboa", "LIS")
	err = tx.Commit()
	assert.Nil(t, err)

	lisbon := City{}
	err = db.Get(&lisbon, "SELECT name FROM City WHERE code = ?", "LIS")
	assert.Nil(t, err)
	assert.Equal(t, "Lisboa", lisbon.Name)
	assert.Equal(t, "", lisbon.Code) // because code column is not part of the select statement

	tx = db.MustBegin()
	tx.MustExec("DELETE FROM City WHERE country_code = ?", "SP")
	err = tx.Commit()
	assert.Nil(t, err)

	cities = []City{}
	err = db.Select(&cities, "SELECT * FROM City WHERE country_code = ?", "SP")
	assert.Nil(t, err)
	assert.Equal(t, 0, len(cities))

	cities = []City{}
	questionMarks, cityCodes := createArrays("OPO", "LIS")
	err = db.Select(&cities, fmt.Sprintf("SELECT * FROM City WHERE code in (%s)", questionMarks), cityCodes...)
	assert.Nil(t, err)
	assert.Equal(t, 2, len(cities))

	paris := City{}
	err = db.Get(&paris, "SELECT * FROM City WHERE code = ?", "PAR")
	assert.NotNil(t, err)
	assert.Equal(t, "", paris.Name)
}

func createArrays(values ...interface{}) (string, []interface{}) {
	var questionMarks []string
	var valueMarks []interface{}
	for _, value := range values {
		questionMarks = append(questionMarks, "?")
		valueMarks = append(valueMarks, value)
	}
	return strings.Join(questionMarks, ","), valueMarks
}
<|endoftext|>


<commit_before>package tree

import "github.com/200sc/go-compgeo/search"

// Type represents the underlying algorithm for updating points on
// a dynamic binary search tree.
// This implementation relies on the idea that, in principle, all
// binary search trees share a lot in common (finding where to
// insert, delete, search), and any remaining details just depend
// on what specific BST type is being used.
type Type int

// TreeType enum
const (
	AVL      Type = iota
	RedBlack      // RB would probably be okay.
	Splay
	//TTree         // T is too short of a variable name
)

// New returns a tree as defined by the input type.
// Hypothetically, this is the only exported function in this package
func New(typ Type) search.Persistable {
	bst := new(BST)
	switch typ {
	case AVL:
		bst.fnSet = avlFnSet
	case Splay:
		bst.fnSet = splayFnSet
	default:
		fallthrough
	case RedBlack:
		bst.fnSet = rbFnSet
	}
	return bst
}
<commit_msg>Added some more potential types (just notes)<commit_after>package tree

import "github.com/200sc/go-compgeo/search"

// Type represents the underlying algorithm for updating points on
// a dynamic binary search tree.
// This implementation relies on the idea that, in principle, all
// binary search trees share a lot in common (finding where to
// insert, delete, search), and any remaining details just depend
// on what specific BST type is being used.
type Type int

// TreeType enum
const (
	AVL      Type = iota
	RedBlack      // RB would probably be okay.
	Splay
	// Consider:
	// Treap?
	// Scapegoat tree?
	// TTree? <- more work than the other two
	// AA?
)

// New returns a tree as defined by the input type.
// Hypothetically, this is the only exported function in this package
func New(typ Type) search.Persistable {
	bst := new(BST)
	switch typ {
	case AVL:
		bst.fnSet = avlFnSet
	case Splay:
		bst.fnSet = splayFnSet
	default:
		fallthrough
	case RedBlack:
		bst.fnSet = rbFnSet
	}
	return bst
}
<|endoftext|>


<commit_before>package markdown

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"

	"github.com/siddontang/ledisdb/config"
	"github.com/siddontang/ledisdb/ledis"
)

var conn *ledis.DB
var GitAddress string
var DbAddress string

type Item struct {
	Key        string
	Title      string
	Desc       string
	Keywords   string
	UpdateTime string
	Content    string
	Tag        string
}

type GitSync struct {
	remote string
	local  string
	tag    string
}

func initDB(DbAddress string) {
	//初始化ledis数据库配置，默认为当前目录下的ledis文件夹中
	var path string
	var dbnumber int
	if len(strings.TrimSpace(DbAddress)) == 0 {
		path = "ledis"
		dbnumber = 0
		createDir(path)
	} else {
		//DbAddress example    /data/ledis?select=1
		path = strings.Split(strings.TrimSpace(DbAddress), "?")[0]
		//如果路径不存在，则创建路径
		if !isDirExist(path) {
			createDir(path)
		}
		var err error
		dbnumber, err = strconv.Atoi(strings.Split(strings.Split(strings.TrimSpace(DbAddress), "?")[1], "=")[1])
		if err != nil {
			dbnumber = 0
		}
	}
	cfg := new(config.Config)
	cfg.DataDir = path
	var err error
	nowLedis, err := ledis.Open(cfg)
	conn, err = nowLedis.Select(dbnumber)
	if err != nil {
		panic(err)
	}
	fmt.Println("....初始化数据库成功....")
}

func initGit(gitAddress string) []*GitSync {
	if len(strings.TrimSpace(gitAddress)) == 0 {
		panic("markdown git地址初始化异常")
	}
	gitAddressArr := strings.Split(strings.TrimSpace(GitAddress), ";")
	gitSyncArr := make([]*GitSync, len(gitAddressArr))
	//将git同步的对象初始化
	for i, _ := range gitAddressArr {
		gitSync := new(GitSync)
		gitSync.remote = strings.Split(gitAddressArr[i], "::")[0]
		gitSync.local = strings.Split(gitAddressArr[i], "::")[1]
		gitSync.tag = strings.Split(gitAddressArr[i], "::")[2]
		gitSyncArr[i] = gitSync
	}
	fmt.Println("....初始化git同步对象成功...", len(gitSyncArr))
	fmt.Printf("%#v\n", gitSyncArr)
	return gitSyncArr
}

func gitClone(remote, local string) {
	fmt.Println("..开始进行克隆操作", "remote=", remote, ";local=", local)
	cmd := exec.Command("git", "clone", remote)
	cmd.Dir = local
	_, err := cmd.Output()
	if err != nil {
		panic(err)
	}
}

func gitPull(local string) {
	fmt.Println("..开始进行pull操作local=", local)
	cmd := exec.Command("git", "pull")
	cmd.Dir = local
	_, err := cmd.Output()
	if err != nil {
		panic(err)
	}
}

func isDirExist(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}

	return fi.IsDir()
}

func createDir(path string) {
	oldMask := syscall.Umask(0)
	os.Mkdir(path, os.ModePerm)
	syscall.Umask(oldMask)
}

func generateDict(tag, path, remote string, syncChan chan string, j int) int {
	fmt.Println(remote, "开始生成目录")
	files, _ := ioutil.ReadDir(path)
	//设定协程数量
	fileMap := make(map[string]*Item, len(files))
	for _, file := range files {
		if file.Name() == "README.md" || file.Name() == "rss.md" || file.Name() == "sitemap.md" || strings.HasPrefix(file.Name(), ".git") {
			continue
		}
		cmd := exec.Command("git", "log", "-1", "--format=\"%ai\"", "--", file.Name())
		cmd.Dir = path
		output, _ := cmd.Output()
		regex, _ := regexp.Compile(`(?m)^"(.*) .*?0800`)
		outputString := string(output)
		result := regex.FindStringSubmatch(outputString)
		var timeString string
		for _, v := range result {
			timeString = v
		}
		item := new(Item)
		item.UpdateTime = timeString
		item.Key = strings.Split(file.Name(), ".")[0]
		fileMap[item.Key] = item
	}
	fmt.Println("....目录在内存中生成完成", "仓库[", j, "]")
	//删除数据库中多余的数据
	DeteleArticle(tag, fileMap)
	//定义线程处理文件插入article
	fileChan := make(chan bool, len(fileMap))
	for i, _ := range fileMap {
		go func(i string) {
			fileMap[i].InsertLedis(tag, path, fileMap[i].Key)
			fileChan <- true
		}(i)
	}
	//
	var i int
	for {
		select {
		case <-fileChan:
			i++
		}
		if i == len(fileMap) {
			syncChan <- fmt.Sprint("....仓库[i", strconv.Itoa(j), "]=", remote, ";数据同步地址=", path, ";数据同步完成,共处理数据", len(fileMap), "条")
			break
		}
	}
	return len(fileMap)
}

func syncAndSave(gitSyncArr []*GitSync, syncChan chan string) {
	fmt.Println("...开始同步git数据...", len(gitSyncArr), "git库的个数为：", gitSyncArr)
	for i, _ := range gitSyncArr {
		go func(ii int) {
			//判断本地路径是否存在，不存在则创建
			fmt.Println(gitSyncArr[ii])
			if !isDirExist(gitSyncArr[ii].local) {
				createDir(gitSyncArr[ii].local)
				gitClone(gitSyncArr[ii].remote, gitSyncArr[ii].local)
				varlength := len(strings.Split(gitSyncArr[ii].remote, "/"))
				//重新复制本地路径local的值，定位到git对应的目录下
				gitSyncArr[ii].local = gitSyncArr[ii].local + "/" + strings.Split(strings.Split(gitSyncArr[ii].remote, "/")[varlength-1], ".")[0]
			} else {
				//判断本地文件夹存在，是否包含所需要的git库
				varlength := len(strings.Split(gitSyncArr[ii].remote, "/"))
				githubRepo := strings.Split(strings.Split(gitSyncArr[ii].remote, "/")[varlength-1], ".")[0]
				//库已经存在
				if repoExist(githubRepo, gitSyncArr[ii].local) {
					gitSyncArr[ii].local = gitSyncArr[ii].local + "/" + strings.Split(strings.Split(gitSyncArr[ii].remote, "/")[varlength-1], ".")[0]
					gitPull(gitSyncArr[ii].local)
				} else {
					//库不存在
					gitClone(gitSyncArr[ii].remote, gitSyncArr[ii].local)
					gitSyncArr[ii].local = gitSyncArr[ii].local + "/" + strings.Split(strings.Split(gitSyncArr[ii].remote, "/")[varlength-1], ".")[0]
				}
			}
			//数据同步完成，开始进行存储
			fmt.Println("仓库[", ii, "]克隆完成")
			generateDict(gitSyncArr[ii].tag, gitSyncArr[ii].local, gitSyncArr[ii].remote, syncChan, ii)
		}(i)
	}
}

func Run() {
	//程序运行结束标志
	end := make(chan bool)
	//初始化ledis数据库连接
	initDB(DbAddress)
	//初始化git仓库
	gitSyncArr := initGit(GitAddress)
	//计数器
	syncChan := make(chan string, len(gitSyncArr))
	var i int
	go func(git string) {
		for {
			select {
			case msg := <-syncChan:
				fmt.Println(msg)
				i++
			}
			if i == len(gitSyncArr) {
				fmt.Println("....所有仓库已经同步完成，并且已经 存入到ledis中....")
				end <- true
				break
			}
		}
	}("wait for syncChan finish")
	//进行git库的同步操作
	syncAndSave(gitSyncArr, syncChan)
	<-end
}

func FindDetail(path, fileName string) (string, string, string, string, string) {
	f, err := os.Open(path + "/" + fileName)
	if err != nil {
		return "", "", "", "", ""
	}
	defer f.Close()
	buff := bufio.NewReader(f)
	content := ""
	title := ""
	desc := ""
	tag := ""
	keywords := ""
	for {
		line, err := buff.ReadString('\n')
		if err != nil || io.EOF == err {
			break
		}
		if strings.HasPrefix(line, "@title:") {
			title = strings.TrimRight(line, "\n")
			title = strings.Replace(title, "@title", "", 1)
			continue
		} else if strings.HasPrefix(line, "@keywords:") {
			keywords = strings.TrimRight(line, "\n")
			keywords = strings.Replace(title, "@keywords:", "", 1)
			continue
		} else if strings.HasPrefix(line, "@desc:") {
			desc = strings.TrimRight(line, "\n")
			desc = strings.Replace(title, "@desc:", "", 1)
			continue
		} else if strings.HasPrefix(line, "@tag:") {
			tag = strings.TrimRight(line, "\n")
			tag = strings.Replace(title, "@tag:", "", 1)
			continue
		}
		content = content + line
	}
	return title, desc, keywords, content, tag
}

func (item *Item) InsertLedis(tag, path, fileName string) {
	title, desc, keywords, content, tag := FindDetail(path, fileName)
	item.Title = title
	item.Keywords = keywords
	item.Content = content
	item.Tag = tag
	item.Desc = desc
	item.Key = strings.Split(fileName, ".")[0]

	//插入ledis 目录
	fmt.Println(tag, item.Key, item.Title+"|"+item.UpdateTime)
	fmt.Println(item.Key, "title", item.Title)
	fmt.Println(item.Key, "desc", item.Desc)
	fmt.Println(item.Key, "keywords", item.Keywords)
	fmt.Println(item.Key, "content", item.Content)

	conn.HSet([]byte(tag), []byte(item.Key), []byte(item.Title+"|"+item.UpdateTime))
	//插入文章内容
	conn.HSet([]byte(item.Key), []byte("title"), []byte(item.Title))
	conn.HSet([]byte(item.Key), []byte("desc"), []byte(item.Desc))
	conn.HSet([]byte(item.Key), []byte("keywords"), []byte(item.Keywords))
	conn.HSet([]byte(item.Key), []byte("content"), []byte(item.Content))
}

//删除掉目录中没有，但是数据库中存在的数据
func DeteleArticle(tag string, fileMap map[string]*Item) {
	fileNames, _ := conn.HKeys([]byte(tag))
	for _, fileName := range fileNames {
		if _, found := fileMap[string(fileName)]; !found {
			//在目录中没有查到该文件，则进行删除
			conn.HDel([]byte(tag), []byte(string(fileName)))
			conn.HDel([]byte(string(fileName)), []byte("title"), []byte("desc"), []byte("keywords"), []byte("content"))
		}
	}
}

func ShowArticleList() {
	fmt.Println("进入展示数据")
	fileNames, _ := conn.HKeys([]byte("A"))
	i := 0
	for _, fileName := range fileNames {
		i = i + 1
		fmt.Println("manage", i)
		data, _ := conn.HGet([]byte("A"), []byte(string(fileName)))
		fmt.Printf("%s\t%s\t%s\n", "B", string(fileName), string(data))
	}
	fileNames, _ = conn.HKeys([]byte("H"))
	j := 0
	for _, fileName := range fileNames {
		j = j + 1
		fmt.Println("manage2", j)
		data, _ := conn.HGet([]byte("B"), []byte(string(fileName)))
		fmt.Printf("%s\t%s\t%s\n", "B", string(fileName), string(data))
	}
}

func repoExist(namespace, path string) bool {
	files, _ := ioutil.ReadDir(path)
	for _, file := range files {
		if file.Name() == namespace {
			return true
		}
	}
	return false
}
<commit_msg>deal with gitrepo<commit_after>package markdown

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"syscall"

	"github.com/siddontang/ledisdb/config"
	"github.com/siddontang/ledisdb/ledis"
)

var conn *ledis.DB
var DbAddress string

var GitAddress string
var Local string
var Tag string

type Item struct {
	Key        string
	Title      string
	Desc       string
	Keywords   string
	UpdateTime string
	Content    string
	Tag        string
}

type GitSync struct {
	remote string
	local  string
	tag    string
}

func initDB(DbAddress string) {
	//初始化ledis数据库配置，默认为当前目录下的ledis文件夹中
	var path string
	var dbnumber int
	if len(strings.TrimSpace(DbAddress)) == 0 {
		path = "ledis"
		dbnumber = 0
		createDir(path)
	} else {
		//DbAddress example    /data/ledis?select=1
		path = strings.Split(strings.TrimSpace(DbAddress), "?")[0]
		//如果路径不存在，则创建路径
		if !isDirExist(path) {
			createDir(path)
		}
		var err error
		dbnumber, err = strconv.Atoi(strings.Split(strings.Split(strings.TrimSpace(DbAddress), "?")[1], "=")[1])
		if err != nil {
			dbnumber = 0
		}
	}
	cfg := new(config.Config)
	cfg.DataDir = path
	var err error
	nowLedis, err := ledis.Open(cfg)
	conn, err = nowLedis.Select(dbnumber)
	if err != nil {
		panic(err)
	}
	fmt.Println("....初始化数据库成功")
}

func initGit(gitAddress, local, tag string) *GitSync {
	if len(strings.TrimSpace(gitAddress)) == 0 || len(strings.TrimSpace(gitAddress)) == 0 || len(strings.TrimSpace(gitAddress)) == 0 {
		panic("....markdown git地址初始化异常")
	}
	//将git同步的对象初始化
	gitSync := new(GitSync)
	gitSync.remote = gitAddress
	gitSync.local = local
	gitSync.tag = tag

	fmt.Println("....初始化git同步对象成功")
	return gitSync
}

func gitClone(remote, local string) {
	fmt.Println("....开始进行克隆操作", "remote=", remote, ";local=", local)
	cmd := exec.Command("git", "clone", remote)
	cmd.Dir = local
	_, err := cmd.Output()
	if err != nil {
		panic(err)
	}
}

func gitPull(local string) {
	fmt.Println("....开始进行pull操作local=", local)
	cmd := exec.Command("git", "pull")
	cmd.Dir = local
	_, err := cmd.Output()
	if err != nil {
		panic(err)
	}
}

func isDirExist(path string) bool {
	fi, err := os.Stat(path)
	if err != nil {
		return false
	}

	return fi.IsDir()
}

func createDir(path string) {
	oldMask := syscall.Umask(0)
	os.Mkdir(path, os.ModePerm)
	syscall.Umask(oldMask)
}

func generateDict(tag, path, remote string) int {
	fmt.Println("....开始在ledis中生成目录")
	files, _ := ioutil.ReadDir(path)
	//设定协程数量
	fileMap := make(map[string]*Item, len(files))
	for _, file := range files {
		if file.Name() == "README.md" || file.Name() == "rss.md" || file.Name() == "sitemap.md" || strings.HasPrefix(file.Name(), ".git") {
			continue
		}
		cmd := exec.Command("git", "log", "-1", "--format=\"%ai\"", "--", file.Name())
		cmd.Dir = path
		output, _ := cmd.Output()
		regex, _ := regexp.Compile(`(?m)^"(.*) .*?0800`)
		outputString := string(output)
		result := regex.FindStringSubmatch(outputString)
		var timeString string
		for _, v := range result {
			timeString = v
		}
		item := new(Item)
		item.UpdateTime = timeString
		item.Key = strings.Split(file.Name(), ".")[0]
		fileMap[item.Key] = item
	}
	//删除数据库中多余的数据
	DeteleArticle(tag, fileMap)
	//定义线程处理文件插入article
	endChan := make(chan string)
	fileChan := make(chan bool, len(fileMap))
	for i, _ := range fileMap {
		go func(i string) {
			fileMap[i].InsertLedis(tag, path, fileMap[i].Key)
			fileChan <- true
		}(i)
	}
	//
	var i int
	go func() {
		for {
			select {
			case <-fileChan:
				i++
			}
			if i == len(fileMap) {
				endChan <- fmt.Sprint("....仓库[", remote, "]", "数据同步地址=", path, ";数据同步完成,共处理数据", len(fileMap), "条")
				break
			}
		}
	}()
	msg := <-endChan
	fmt.Println(msg)
	return len(fileMap)
}

func syncAndSave(gitSync *GitSync) {
	fmt.Println("....开始同步git数据")
	//判断本地路径是否存在，不存在则创建
	if !isDirExist(gitSync.local) {
		createDir(gitSync.local)
		gitClone(gitSync.remote, gitSync.local)
		varlength := len(strings.Split(gitSync.remote, "/"))
		//重新复制本地路径local的值，定位到git对应的目录下
		gitSync.local = gitSync.local + "/" + strings.Split(strings.Split(gitSync.remote, "/")[varlength-1], ".")[0]
	} else {
		//判断本地文件夹存在，是否包含所需要的git库
		varlength := len(strings.Split(gitSync.remote, "/"))
		githubRepo := strings.Split(strings.Split(gitSync.remote, "/")[varlength-1], ".")[0]
		//库已经存在
		if repoExist(githubRepo, gitSync.local) {
			gitSync.local = gitSync.local + "/" + strings.Split(strings.Split(gitSync.remote, "/")[varlength-1], ".")[0]
			gitPull(gitSync.local)
		} else {
			//库不存在
			gitClone(gitSync.remote, gitSync.local)
			gitSync.local = gitSync.local + "/" + strings.Split(strings.Split(gitSync.remote, "/")[varlength-1], ".")[0]
		}
	}
	//数据同步完成，开始进行存储
	fmt.Println("....仓库[", gitSync.remote, "]同步本地完成,准备插入数据到ledis中")
	generateDict(gitSync.tag, gitSync.local, gitSync.remote)
}

func Run() {
	//初始化ledis数据库连接
	initDB(DbAddress)
	//初始化git仓库
	gitSync := initGit(GitAddress, Local, Tag)

	//进行git库的同步操作
	syncAndSave(gitSync)
}

func FindDetail(path, fileName string) (string, string, string, string, string) {
	f, err := os.Open(path + "/" + fileName)
	if err != nil {
		return "", "", "", "", ""
	}
	defer f.Close()
	buff := bufio.NewReader(f)
	content := ""
	title := ""
	desc := ""
	tag := ""
	keywords := ""
	for {
		line, err := buff.ReadString('\n')
		if err != nil || io.EOF == err {
			break
		}
		if strings.HasPrefix(line, "@title:") {
			title = strings.TrimRight(line, "\n")
			title = strings.Replace(title, "@title", "", 1)
			continue
		} else if strings.HasPrefix(line, "@keywords:") {
			keywords = strings.TrimRight(line, "\n")
			keywords = strings.Replace(title, "@keywords:", "", 1)
			continue
		} else if strings.HasPrefix(line, "@desc:") {
			desc = strings.TrimRight(line, "\n")
			desc = strings.Replace(title, "@desc:", "", 1)
			continue
		} else if strings.HasPrefix(line, "@tag:") {
			tag = strings.TrimRight(line, "\n")
			tag = strings.Replace(title, "@tag:", "", 1)
			continue
		}
		content = content + line
	}
	return title, desc, keywords, content, tag
}

func (item *Item) InsertLedis(tag, path, fileName string) {
	title, desc, keywords, content, tag := FindDetail(path, fileName)
	item.Title = title
	item.Keywords = keywords
	item.Content = content
	item.Tag = tag
	item.Desc = desc
	item.Key = strings.Split(fileName, ".")[0]

	//插入ledis 目录
	fmt.Println(tag, item.Key, item.Title+"|"+item.UpdateTime)
	fmt.Println(item.Key, "title", item.Title)
	fmt.Println(item.Key, "desc", item.Desc)
	fmt.Println(item.Key, "keywords", item.Keywords)
	fmt.Println(item.Key, "content", item.Content)

	conn.HSet([]byte(tag), []byte(item.Key), []byte(item.Title+"|"+item.UpdateTime))
	//插入文章内容
	conn.HSet([]byte(item.Key), []byte("title"), []byte(item.Title))
	conn.HSet([]byte(item.Key), []byte("desc"), []byte(item.Desc))
	conn.HSet([]byte(item.Key), []byte("keywords"), []byte(item.Keywords))
	conn.HSet([]byte(item.Key), []byte("content"), []byte(item.Content))
}

//删除掉目录中没有，但是数据库中存在的数据
func DeteleArticle(tag string, fileMap map[string]*Item) {
	fileNames, _ := conn.HKeys([]byte(tag))
	for _, fileName := range fileNames {
		if _, found := fileMap[string(fileName)]; !found {
			//在目录中没有查到该文件，则进行删除
			conn.HDel([]byte(tag), []byte(string(fileName)))
			conn.HDel([]byte(string(fileName)), []byte("title"), []byte("desc"), []byte("keywords"), []byte("content"))
		}
	}
}

func ShowArticleList() {
	fmt.Println("进入展示数据")
	fileNames, _ := conn.HKeys([]byte("A"))
	i := 0
	for _, fileName := range fileNames {
		i = i + 1
		fmt.Println("manage", i)
		data, _ := conn.HGet([]byte("A"), []byte(string(fileName)))
		fmt.Printf("%s\t%s\t%s\n", "B", string(fileName), string(data))
	}
	fileNames, _ = conn.HKeys([]byte("H"))
	j := 0
	for _, fileName := range fileNames {
		j = j + 1
		fmt.Println("manage2", j)
		data, _ := conn.HGet([]byte("B"), []byte(string(fileName)))
		fmt.Printf("%s\t%s\t%s\n", "B", string(fileName), string(data))
	}
}

func repoExist(namespace, path string) bool {
	files, _ := ioutil.ReadDir(path)
	for _, file := range files {
		if file.Name() == namespace {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package encoders

import (
	"errors"
	"fmt"
	"strconv"

	"code.cloudfoundry.org/lager"

	"github.com/3d0c/gmf"
	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"
)

// FFMPEGEncode function is responsible for encoding the file
func FFMPEGEncode(logger lager.Logger, dbInstance db.Storage, jobID string) error {
	log := logger.Session("ffmpeg-encode")
	log.Info("started", lager.Data{"job": jobID})
	defer log.Info("finished")

	gmf.LogSetLevel(gmf.AV_LOG_FATAL)
	job, _ := dbInstance.RetrieveJob(jobID)

	// create input context
	inputCtx, err := gmf.NewInputCtx(job.LocalSource)
	if err != nil {
		log.Error("input-failed", err)
		return err
	}
	defer inputCtx.CloseInputAndRelease()

	// create output context
	outputCtx, err := gmf.NewOutputCtx(job.LocalDestination)
	if err != nil {
		log.Error("output-failed", err)
		return err
	}
	defer outputCtx.CloseOutputAndRelease()

	job.Status = types.JobEncoding
	job.Progress = "0%"
	dbInstance.UpdateJob(job.ID, job)

	//get audio and video stream and the streaMap
	streamMap, srcVideoStream, srcAudioStream, err := getAudioVideoStreamSource(inputCtx, outputCtx, job)
	if err != nil {
		return err
	}
	//calculate total number of frames
	totalFrames := float64(srcVideoStream.NbFrames() + srcAudioStream.NbFrames())
	//process all frames and update the job progress
	err = processAllFramesAndUpdateJobProgress(inputCtx, outputCtx, streamMap, job, dbInstance, totalFrames)
	if err != nil {
		return err
	}

	processNewFrames(inputCtx, outputCtx, streamMap)
	if err != nil {
		return err
	}

	if job.Progress != "100%" {
		job.Progress = "100%"
		dbInstance.UpdateJob(job.ID, job)
	}

	return nil
}

func processNewFrames(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, streamMap map[int]int) error {
	for i := 0; i < outputCtx.StreamsCnt(); i++ {
		inputStream, err := getStream(inputCtx, 0)
		if err != nil {
			return err
		}
		outputStream, err := getStream(outputCtx, streamMap[inputStream.Index()])
		if err != nil {
			return err
		}

		frame := gmf.NewFrame()

		for {
			if p, ready, _ := frame.FlushNewPacket(outputStream.CodecCtx()); ready {
				configurePacket(p, outputStream, frame)
				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			} else {
				break
			}
			outputStream.Pts++
		}

		gmf.Release(frame)
	}

	return nil
}

func processAllFramesAndUpdateJobProgress(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, streamMap map[int]int, job types.Job, dbInstance db.Storage, totalFrames float64) error {
	var lastDelta int64
	framesCount := float64(0)
	for packet := range inputCtx.GetNewPackets() {
		inputStream, err := getStream(inputCtx, packet.StreamIndex())
		if err != nil {
			return err
		}
		outputStream, err := getStream(outputCtx, streamMap[inputStream.Index()])
		if err != nil {
			return err
		}

		for frame := range packet.Frames(inputStream.CodecCtx()) {
			err := processFrame(inputStream, outputStream, packet, frame, outputCtx, &lastDelta)
			if err != nil {
				return err
			}

			outputStream.Pts++
			framesCount++
			percentage := fmt.Sprintf("%.2f", framesCount/totalFrames*100) + "%"
			if percentage != job.Progress {
				job.Progress = percentage
				dbInstance.UpdateJob(job.ID, job)
			}
		}

		gmf.Release(packet)
	}
	return nil
}

func getStream(context *gmf.FmtCtx, streamIndex int) (*gmf.Stream, error) {
	return context.GetStream(streamIndex)
}

func getAudioVideoStreamSource(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, job types.Job) (map[int]int, *gmf.Stream, *gmf.Stream, error) {
	streamMap := make(map[int]int, 0)

	// add video stream to streamMap
	srcVideoStream, err := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_VIDEO)
	if err != nil {
		return nil, nil, nil, errors.New("unable to get the best video stream inside the input context")
	}
	videoCodec := getVideoCodec(job)
	inputIndex, outputIndex, err := addStream(job, videoCodec, outputCtx, srcVideoStream)
	if err != nil {
		return nil, nil, nil, err
	}
	streamMap[inputIndex] = outputIndex

	// add audio stream to streamMap
	srcAudioStream, err := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_AUDIO)
	if err != nil {
		return nil, nil, nil, errors.New("unable to get the best audio stream inside the input context")
	}
	audioCodec := getAudioCodec(job)
	inputIndex, outputIndex, err = addStream(job, audioCodec, outputCtx, srcAudioStream)
	if err != nil {
		return nil, nil, nil, err
	}
	streamMap[inputIndex] = outputIndex
	if err := outputCtx.WriteHeader(); err != nil {
		return nil, nil, nil, err
	}

	return streamMap, srcVideoStream, srcAudioStream, nil
}

func configureAudioFrame(packet *gmf.Packet, inputStream *gmf.Stream, outputStream *gmf.Stream, frame *gmf.Frame, lastDelta *int64) {
	fsTb := gmf.AVR{Num: 1, Den: inputStream.CodecCtx().SampleRate()}
	outTb := gmf.AVR{Num: 1, Den: inputStream.CodecCtx().SampleRate()}

	frame.SetPts(packet.Pts())

	pts := gmf.RescaleDelta(inputStream.TimeBase(), frame.Pts(), fsTb.AVRational(), frame.NbSamples(), lastDelta, outTb.AVRational())

	frame.SetNbSamples(outputStream.CodecCtx().FrameSize())
	frame.SetFormat(outputStream.CodecCtx().SampleFmt())
	frame.SetChannelLayout(outputStream.CodecCtx().ChannelLayout())
	frame.SetPts(pts)
}

func configurePacket(packet *gmf.Packet, outputStream *gmf.Stream, frame *gmf.Frame) *gmf.Packet {
	if packet.Pts() != gmf.AV_NOPTS_VALUE {
		packet.SetPts(gmf.RescaleQ(packet.Pts(), outputStream.CodecCtx().TimeBase(), outputStream.TimeBase()))
	}

	if packet.Dts() != gmf.AV_NOPTS_VALUE {
		packet.SetDts(gmf.RescaleQ(packet.Dts(), outputStream.CodecCtx().TimeBase(), outputStream.TimeBase()))
	}

	packet.SetStreamIndex(outputStream.Index())

	return packet
}

func processFrame(inputStream *gmf.Stream, outputStream *gmf.Stream, packet *gmf.Packet, frame *gmf.Frame, outputCtx *gmf.FmtCtx, lastDelta *int64) error {
	if outputStream.IsAudio() {
		configureAudioFrame(packet, inputStream, outputStream, frame, lastDelta)
	} else {
		frame.SetPts(outputStream.Pts)
	}

	if newPacket, ready, _ := frame.EncodeNewPacket(outputStream.CodecCtx()); ready {
		configurePacket(newPacket, outputStream, frame)
		if err := outputCtx.WritePacket(newPacket); err != nil {
			return err
		}
		gmf.Release(newPacket)
	}

	return nil
}

func addStream(job types.Job, codecName string, oc *gmf.FmtCtx, inputStream *gmf.Stream) (int, int, error) {
	var codecContext *gmf.CodecCtx
	var outputStream *gmf.Stream

	codec, err := gmf.FindEncoder(codecName)
	if err != nil {
		return 0, 0, err
	}

	if outputStream = oc.NewStream(codec); outputStream == nil {
		return 0, 0, errors.New("unable to create stream in output context")
	}
	defer gmf.Release(outputStream)

	if codecContext = gmf.NewCodecCtx(codec); codecContext == nil {
		return 0, 0, errors.New("unable to create codec context")
	}
	defer gmf.Release(codecContext)

	// https://ffmpeg.org/pipermail/ffmpeg-devel/2008-January/046900.html
	if oc.IsGlobalHeader() {
		codecContext.SetFlag(gmf.CODEC_FLAG_GLOBAL_HEADER)
	}

	if codec.IsExperimental() {
		codecContext.SetStrictCompliance(gmf.FF_COMPLIANCE_EXPERIMENTAL)
	}

	if codecContext.Type() == gmf.AVMEDIA_TYPE_AUDIO {
		err := setAudioCtxParams(codecContext, inputStream, job)
		if err != nil {
			return 0, 0, err
		}
	}

	if codecContext.Type() == gmf.AVMEDIA_TYPE_VIDEO {
		err := setVideoCtxParams(codecContext, inputStream, job)
		if err != nil {
			return 0, 0, err
		}
	}

	if err := codecContext.Open(nil); err != nil {
		return 0, 0, err
	}

	outputStream.SetCodecCtx(codecContext)

	return inputStream.Index(), outputStream.Index(), nil
}

func getProfile(job types.Job) int {
	profiles := map[string]int{
		"baseline": gmf.FF_PROFILE_H264_BASELINE,
		"main":     gmf.FF_PROFILE_H264_MAIN,
		"high":     gmf.FF_PROFILE_H264_HIGH,
	}

	if job.Preset.Video.Profile != "" {
		return profiles[job.Preset.Video.Profile]
	}
	return gmf.FF_PROFILE_H264_MAIN
}

func getVideoCodec(job types.Job) string {
	codecs := map[string]string{
		"h264":   "libx264",
		"vp8":    "libvpx",
		"vp9":    "libvpx-vp9",
		"theora": "libtheora",
		"aac":    "aac",
	}

	if codec, ok := codecs[job.Preset.Video.Codec]; ok {
		return codec
	}
	return "libx264"
}

func getAudioCodec(job types.Job) string {
	codecs := map[string]string{
		"aac":    "aac",
		"vorbis": "vorbis",
	}
	if codec, ok := codecs[job.Preset.Audio.Codec]; ok {
		return codec
	}
	return "aac"
}

func getResolution(job types.Job, inputWidth int, inputHeight int) (int, int) {
	var width, height int
	if job.Preset.Video.Width == "" && job.Preset.Video.Height == "" {
		return inputWidth, inputHeight
	} else if job.Preset.Video.Width == "" {
		height, _ = strconv.Atoi(job.Preset.Video.Height)
		width = (inputWidth * height) / inputHeight
	} else if job.Preset.Video.Height == "" {
		width, _ = strconv.Atoi(job.Preset.Video.Width)
		height = (inputHeight * width) / inputWidth
	} else {
		width, _ = strconv.Atoi(job.Preset.Video.Width)
		height, _ = strconv.Atoi(job.Preset.Video.Height)
	}
	return width, height
}

func setAudioCtxParams(codecContext *gmf.CodecCtx, ist *gmf.Stream, job types.Job) error {
	bitrate, err := strconv.Atoi(job.Preset.Audio.Bitrate)
	if err != nil {
		return err
	}

	codecContext.SetBitRate(bitrate)
	codecContext.SetSampleFmt(ist.CodecCtx().SampleFmt())
	codecContext.SetSampleRate(ist.CodecCtx().SampleRate())
	codecContext.SetChannels(ist.CodecCtx().Channels())
	codecContext.SelectChannelLayout()
	codecContext.SelectSampleRate()
	return nil
}

func setVideoCtxParams(codecContext *gmf.CodecCtx, ist *gmf.Stream, job types.Job) error {
	codecContext.SetTimeBase(gmf.AVR{Num: 1, Den: 25}) // what is this

	if job.Preset.Video.Codec == "h264" {
		profile := getProfile(job)
		codecContext.SetProfile(profile)
	}

	gop, err := strconv.Atoi(job.Preset.Video.GopSize)
	if err != nil {
		return err
	}

	width, height := getResolution(job, ist.CodecCtx().Width(), ist.CodecCtx().Height())

	bitrate, err := strconv.Atoi(job.Preset.Video.Bitrate)
	if err != nil {
		return err
	}

	codecContext.SetDimension(width, height)
	codecContext.SetGopSize(gop)
	codecContext.SetBitRate(bitrate)
	codecContext.SetPixFmt(ist.CodecCtx().PixFmt())

	return nil
}
<commit_msg>Missing returned error<commit_after>package encoders

import (
	"errors"
	"fmt"
	"strconv"

	"code.cloudfoundry.org/lager"

	"github.com/3d0c/gmf"
	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"
)

// FFMPEGEncode function is responsible for encoding the file
func FFMPEGEncode(logger lager.Logger, dbInstance db.Storage, jobID string) error {
	log := logger.Session("ffmpeg-encode")
	log.Info("started", lager.Data{"job": jobID})
	defer log.Info("finished")

	gmf.LogSetLevel(gmf.AV_LOG_FATAL)
	job, _ := dbInstance.RetrieveJob(jobID)

	// create input context
	inputCtx, err := gmf.NewInputCtx(job.LocalSource)
	if err != nil {
		log.Error("input-failed", err)
		return err
	}
	defer inputCtx.CloseInputAndRelease()

	// create output context
	outputCtx, err := gmf.NewOutputCtx(job.LocalDestination)
	if err != nil {
		log.Error("output-failed", err)
		return err
	}
	defer outputCtx.CloseOutputAndRelease()

	job.Status = types.JobEncoding
	job.Progress = "0%"
	dbInstance.UpdateJob(job.ID, job)

	//get audio and video stream and the streaMap
	streamMap, srcVideoStream, srcAudioStream, err := getAudioVideoStreamSource(inputCtx, outputCtx, job)
	if err != nil {
		return err
	}
	//calculate total number of frames
	totalFrames := float64(srcVideoStream.NbFrames() + srcAudioStream.NbFrames())
	//process all frames and update the job progress
	err = processAllFramesAndUpdateJobProgress(inputCtx, outputCtx, streamMap, job, dbInstance, totalFrames)
	if err != nil {
		return err
	}

	err = processNewFrames(inputCtx, outputCtx, streamMap)
	if err != nil {
		return err
	}

	if job.Progress != "100%" {
		job.Progress = "100%"
		dbInstance.UpdateJob(job.ID, job)
	}

	return nil
}

func processNewFrames(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, streamMap map[int]int) error {
	for i := 0; i < outputCtx.StreamsCnt(); i++ {
		inputStream, err := getStream(inputCtx, 0)
		if err != nil {
			return err
		}
		outputStream, err := getStream(outputCtx, streamMap[inputStream.Index()])
		if err != nil {
			return err
		}

		frame := gmf.NewFrame()

		for {
			if p, ready, _ := frame.FlushNewPacket(outputStream.CodecCtx()); ready {
				configurePacket(p, outputStream, frame)
				if err := outputCtx.WritePacket(p); err != nil {
					return err
				}
				gmf.Release(p)
			} else {
				break
			}
			outputStream.Pts++
		}

		gmf.Release(frame)
	}

	return nil
}

func processAllFramesAndUpdateJobProgress(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, streamMap map[int]int, job types.Job, dbInstance db.Storage, totalFrames float64) error {
	var lastDelta int64
	framesCount := float64(0)
	for packet := range inputCtx.GetNewPackets() {
		inputStream, err := getStream(inputCtx, packet.StreamIndex())
		if err != nil {
			return err
		}
		outputStream, err := getStream(outputCtx, streamMap[inputStream.Index()])
		if err != nil {
			return err
		}

		for frame := range packet.Frames(inputStream.CodecCtx()) {
			err := processFrame(inputStream, outputStream, packet, frame, outputCtx, &lastDelta)
			if err != nil {
				return err
			}

			outputStream.Pts++
			framesCount++
			percentage := fmt.Sprintf("%.2f", framesCount/totalFrames*100) + "%"
			if percentage != job.Progress {
				job.Progress = percentage
				dbInstance.UpdateJob(job.ID, job)
			}
		}

		gmf.Release(packet)
	}
	return nil
}

func getStream(context *gmf.FmtCtx, streamIndex int) (*gmf.Stream, error) {
	return context.GetStream(streamIndex)
}

func getAudioVideoStreamSource(inputCtx *gmf.FmtCtx, outputCtx *gmf.FmtCtx, job types.Job) (map[int]int, *gmf.Stream, *gmf.Stream, error) {
	streamMap := make(map[int]int, 0)

	// add video stream to streamMap
	srcVideoStream, err := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_VIDEO)
	if err != nil {
		return nil, nil, nil, errors.New("unable to get the best video stream inside the input context")
	}
	videoCodec := getVideoCodec(job)
	inputIndex, outputIndex, err := addStream(job, videoCodec, outputCtx, srcVideoStream)
	if err != nil {
		return nil, nil, nil, err
	}
	streamMap[inputIndex] = outputIndex

	// add audio stream to streamMap
	srcAudioStream, err := inputCtx.GetBestStream(gmf.AVMEDIA_TYPE_AUDIO)
	if err != nil {
		return nil, nil, nil, errors.New("unable to get the best audio stream inside the input context")
	}
	audioCodec := getAudioCodec(job)
	inputIndex, outputIndex, err = addStream(job, audioCodec, outputCtx, srcAudioStream)
	if err != nil {
		return nil, nil, nil, err
	}
	streamMap[inputIndex] = outputIndex
	if err := outputCtx.WriteHeader(); err != nil {
		return nil, nil, nil, err
	}

	return streamMap, srcVideoStream, srcAudioStream, nil
}

func configureAudioFrame(packet *gmf.Packet, inputStream *gmf.Stream, outputStream *gmf.Stream, frame *gmf.Frame, lastDelta *int64) {
	fsTb := gmf.AVR{Num: 1, Den: inputStream.CodecCtx().SampleRate()}
	outTb := gmf.AVR{Num: 1, Den: inputStream.CodecCtx().SampleRate()}

	frame.SetPts(packet.Pts())

	pts := gmf.RescaleDelta(inputStream.TimeBase(), frame.Pts(), fsTb.AVRational(), frame.NbSamples(), lastDelta, outTb.AVRational())

	frame.SetNbSamples(outputStream.CodecCtx().FrameSize())
	frame.SetFormat(outputStream.CodecCtx().SampleFmt())
	frame.SetChannelLayout(outputStream.CodecCtx().ChannelLayout())
	frame.SetPts(pts)
}

func configurePacket(packet *gmf.Packet, outputStream *gmf.Stream, frame *gmf.Frame) *gmf.Packet {
	if packet.Pts() != gmf.AV_NOPTS_VALUE {
		packet.SetPts(gmf.RescaleQ(packet.Pts(), outputStream.CodecCtx().TimeBase(), outputStream.TimeBase()))
	}

	if packet.Dts() != gmf.AV_NOPTS_VALUE {
		packet.SetDts(gmf.RescaleQ(packet.Dts(), outputStream.CodecCtx().TimeBase(), outputStream.TimeBase()))
	}

	packet.SetStreamIndex(outputStream.Index())

	return packet
}

func processFrame(inputStream *gmf.Stream, outputStream *gmf.Stream, packet *gmf.Packet, frame *gmf.Frame, outputCtx *gmf.FmtCtx, lastDelta *int64) error {
	if outputStream.IsAudio() {
		configureAudioFrame(packet, inputStream, outputStream, frame, lastDelta)
	} else {
		frame.SetPts(outputStream.Pts)
	}

	if newPacket, ready, _ := frame.EncodeNewPacket(outputStream.CodecCtx()); ready {
		configurePacket(newPacket, outputStream, frame)
		if err := outputCtx.WritePacket(newPacket); err != nil {
			return err
		}
		gmf.Release(newPacket)
	}

	return nil
}

func addStream(job types.Job, codecName string, oc *gmf.FmtCtx, inputStream *gmf.Stream) (int, int, error) {
	var codecContext *gmf.CodecCtx
	var outputStream *gmf.Stream

	codec, err := gmf.FindEncoder(codecName)
	if err != nil {
		return 0, 0, err
	}

	if outputStream = oc.NewStream(codec); outputStream == nil {
		return 0, 0, errors.New("unable to create stream in output context")
	}
	defer gmf.Release(outputStream)

	if codecContext = gmf.NewCodecCtx(codec); codecContext == nil {
		return 0, 0, errors.New("unable to create codec context")
	}
	defer gmf.Release(codecContext)

	// https://ffmpeg.org/pipermail/ffmpeg-devel/2008-January/046900.html
	if oc.IsGlobalHeader() {
		codecContext.SetFlag(gmf.CODEC_FLAG_GLOBAL_HEADER)
	}

	if codec.IsExperimental() {
		codecContext.SetStrictCompliance(gmf.FF_COMPLIANCE_EXPERIMENTAL)
	}

	if codecContext.Type() == gmf.AVMEDIA_TYPE_AUDIO {
		err := setAudioCtxParams(codecContext, inputStream, job)
		if err != nil {
			return 0, 0, err
		}
	}

	if codecContext.Type() == gmf.AVMEDIA_TYPE_VIDEO {
		err := setVideoCtxParams(codecContext, inputStream, job)
		if err != nil {
			return 0, 0, err
		}
	}

	if err := codecContext.Open(nil); err != nil {
		return 0, 0, err
	}

	outputStream.SetCodecCtx(codecContext)

	return inputStream.Index(), outputStream.Index(), nil
}

func getProfile(job types.Job) int {
	profiles := map[string]int{
		"baseline": gmf.FF_PROFILE_H264_BASELINE,
		"main":     gmf.FF_PROFILE_H264_MAIN,
		"high":     gmf.FF_PROFILE_H264_HIGH,
	}

	if job.Preset.Video.Profile != "" {
		return profiles[job.Preset.Video.Profile]
	}
	return gmf.FF_PROFILE_H264_MAIN
}

func getVideoCodec(job types.Job) string {
	codecs := map[string]string{
		"h264":   "libx264",
		"vp8":    "libvpx",
		"vp9":    "libvpx-vp9",
		"theora": "libtheora",
		"aac":    "aac",
	}

	if codec, ok := codecs[job.Preset.Video.Codec]; ok {
		return codec
	}
	return "libx264"
}

func getAudioCodec(job types.Job) string {
	codecs := map[string]string{
		"aac":    "aac",
		"vorbis": "vorbis",
	}
	if codec, ok := codecs[job.Preset.Audio.Codec]; ok {
		return codec
	}
	return "aac"
}

func getResolution(job types.Job, inputWidth int, inputHeight int) (int, int) {
	var width, height int
	if job.Preset.Video.Width == "" && job.Preset.Video.Height == "" {
		return inputWidth, inputHeight
	} else if job.Preset.Video.Width == "" {
		height, _ = strconv.Atoi(job.Preset.Video.Height)
		width = (inputWidth * height) / inputHeight
	} else if job.Preset.Video.Height == "" {
		width, _ = strconv.Atoi(job.Preset.Video.Width)
		height = (inputHeight * width) / inputWidth
	} else {
		width, _ = strconv.Atoi(job.Preset.Video.Width)
		height, _ = strconv.Atoi(job.Preset.Video.Height)
	}
	return width, height
}

func setAudioCtxParams(codecContext *gmf.CodecCtx, ist *gmf.Stream, job types.Job) error {
	bitrate, err := strconv.Atoi(job.Preset.Audio.Bitrate)
	if err != nil {
		return err
	}

	codecContext.SetBitRate(bitrate)
	codecContext.SetSampleFmt(ist.CodecCtx().SampleFmt())
	codecContext.SetSampleRate(ist.CodecCtx().SampleRate())
	codecContext.SetChannels(ist.CodecCtx().Channels())
	codecContext.SelectChannelLayout()
	codecContext.SelectSampleRate()
	return nil
}

func setVideoCtxParams(codecContext *gmf.CodecCtx, ist *gmf.Stream, job types.Job) error {
	codecContext.SetTimeBase(gmf.AVR{Num: 1, Den: 25}) // what is this

	if job.Preset.Video.Codec == "h264" {
		profile := getProfile(job)
		codecContext.SetProfile(profile)
	}

	gop, err := strconv.Atoi(job.Preset.Video.GopSize)
	if err != nil {
		return err
	}

	width, height := getResolution(job, ist.CodecCtx().Width(), ist.CodecCtx().Height())

	bitrate, err := strconv.Atoi(job.Preset.Video.Bitrate)
	if err != nil {
		return err
	}

	codecContext.SetDimension(width, height)
	codecContext.SetGopSize(gop)
	codecContext.SetBitRate(bitrate)
	codecContext.SetPixFmt(ist.CodecCtx().PixFmt())

	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/jkomoros/boardgame"
	"log"
	"strconv"
	"time"
)

const (
	maxMessageSize = 512
	writeWait      = 10 * time.Second
	pongWait       = 60 * time.Second
	pingPeriod     = (pongWait * 9) / 10
)

type gameVersionChanged struct {
	Id      string
	Version int
}

type versionNotifier struct {
	sockets       map[string]map[*socket]bool
	register      chan *socket
	unregister    chan *socket
	notifyVersion chan gameVersionChanged
	doneChan      chan bool
}

type socket struct {
	gameId   string
	notifier *versionNotifier
	conn     *websocket.Conn
	send     chan []byte
}

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func (s *Server) socketHandler(c *gin.Context) {
	game := s.getGame(c)

	renderer := NewRenderer(c)

	if game == nil {
		renderer.Error("No such game")
	}

	conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)

	if err != nil {
		renderer.Error("Couldn't upgrade socket: " + err.Error())
	}

	socket := newSocket(game.Id(), conn, s.notifier)
	s.notifier.register <- socket

}

func newSocket(gameId string, conn *websocket.Conn, notifier *versionNotifier) *socket {
	result := &socket{
		notifier: notifier,
		conn:     conn,
		send:     make(chan []byte, 256),
		gameId:   gameId,
	}
	go result.readPump()
	go result.writePump()

	return result
}

func (s *socket) readPump() {

	//Based on implementation from https://github.com/gorilla/websocket/blob/master/examples/chat/client.go

	defer func() {
		s.notifier.unregister <- s
		s.conn.Close()
	}()

	s.conn.SetReadLimit(maxMessageSize)
	s.conn.SetReadDeadline(time.Now().Add(pongWait))
	s.conn.SetPongHandler(func(string) error { s.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })

	for {
		_, message, err := s.conn.ReadMessage()
		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				log.Printf("error: %v", err)
			}
			break
		}
		log.Println("Unexpectedly got a message: ", message)
	}

}

func (s *socket) writePump() {

	//Based on implementation at https://github.com/gorilla/websocket/blob/master/examples/chat/client.go

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		s.conn.Close()
	}()
	for {
		select {
		case message, ok := <-s.send:
			s.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if !ok {
				// The hub closed the channel.
				s.conn.WriteMessage(websocket.CloseMessage, []byte{})
				return
			}

			s.conn.WriteMessage(websocket.TextMessage, message)
		case <-ticker.C:
			s.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if err := s.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}

}

func newVersionNotifier() *versionNotifier {
	result := &versionNotifier{
		sockets:       make(map[string]map[*socket]bool),
		register:      make(chan *socket),
		unregister:    make(chan *socket),
		notifyVersion: make(chan gameVersionChanged),
		doneChan:      make(chan bool),
	}
	go result.workLoop()
	return result
}

func (v *versionNotifier) gameChanged(game *boardgame.Game) {
	v.notifyVersion <- gameVersionChanged{
		Id:      game.Id(),
		Version: game.Version(),
	}
}

func (v *versionNotifier) done() {
	close(v.doneChan)
}

func (v *versionNotifier) workLoop() {
	for {
		select {
		case s := <-v.register:
			v.registerSocket(s)
		case s := <-v.unregister:
			v.unregisterSocket(s)
		case rec := <-v.notifyVersion:
			//Send message
			bucket, ok := v.sockets[rec.Id]
			if ok {
				//Someone's listening!
				for socket := range bucket {
					socket.send <- []byte(strconv.Itoa(rec.Version))
				}
			}
		case <-v.doneChan:
			break
		}
	}
}

func (v *versionNotifier) registerSocket(s *socket) {
	//Should only be called by workLoop

	bucket, ok := v.sockets[s.gameId]

	if !ok {
		bucket = make(map[*socket]bool)
		v.sockets[s.gameId] = bucket
	}

	bucket[s] = true
}

func (v *versionNotifier) unregisterSocket(s *socket) {
	//Should only be called by workloop

	bucket, ok := v.sockets[s.gameId]

	if !ok {
		return
	}

	delete(bucket, s)
}
<commit_msg>Add debug messages to server web socket code. Part of #401.<commit_after>package api

import (
	"github.com/gin-gonic/gin"
	"github.com/gorilla/websocket"
	"github.com/jkomoros/boardgame"
	"log"
	"strconv"
	"time"
)

const (
	maxMessageSize = 512
	writeWait      = 10 * time.Second
	pongWait       = 60 * time.Second
	pingPeriod     = (pongWait * 9) / 10
)

const debugSockets = true

type gameVersionChanged struct {
	Id      string
	Version int
}

type versionNotifier struct {
	sockets       map[string]map[*socket]bool
	register      chan *socket
	unregister    chan *socket
	notifyVersion chan gameVersionChanged
	doneChan      chan bool
}

type socket struct {
	gameId   string
	notifier *versionNotifier
	conn     *websocket.Conn
	send     chan []byte
}

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func (s *Server) socketHandler(c *gin.Context) {
	game := s.getGame(c)

	renderer := NewRenderer(c)

	if game == nil {
		renderer.Error("No such game")
	}

	conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)

	if err != nil {
		renderer.Error("Couldn't upgrade socket: " + err.Error())
	}

	socket := newSocket(game.Id(), conn, s.notifier)
	s.notifier.register <- socket

}

func newSocket(gameId string, conn *websocket.Conn, notifier *versionNotifier) *socket {
	result := &socket{
		notifier: notifier,
		conn:     conn,
		send:     make(chan []byte, 256),
		gameId:   gameId,
	}
	go result.readPump()
	go result.writePump()

	return result
}

func (s *socket) readPump() {

	//Based on implementation from https://github.com/gorilla/websocket/blob/master/examples/chat/client.go

	defer func() {
		s.notifier.unregister <- s
		s.conn.Close()
	}()

	s.conn.SetReadLimit(maxMessageSize)
	s.conn.SetReadDeadline(time.Now().Add(pongWait))
	s.conn.SetPongHandler(func(string) error { s.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })

	for {
		_, message, err := s.conn.ReadMessage()
		if err != nil {
			if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway) {
				log.Printf("error: %v", err)
			}
			break
		}
		log.Println("Unexpectedly got a message: ", message)
	}

}

func (s *socket) writePump() {

	//Based on implementation at https://github.com/gorilla/websocket/blob/master/examples/chat/client.go

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		s.conn.Close()
	}()
	for {
		select {
		case message, ok := <-s.send:
			s.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if !ok {
				// The hub closed the channel.
				s.conn.WriteMessage(websocket.CloseMessage, []byte{})
				return
			}

			s.conn.WriteMessage(websocket.TextMessage, message)
		case <-ticker.C:
			s.conn.SetWriteDeadline(time.Now().Add(writeWait))
			if err := s.conn.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}

}

func newVersionNotifier() *versionNotifier {
	result := &versionNotifier{
		sockets:       make(map[string]map[*socket]bool),
		register:      make(chan *socket),
		unregister:    make(chan *socket),
		notifyVersion: make(chan gameVersionChanged),
		doneChan:      make(chan bool),
	}
	go result.workLoop()
	return result
}

func (v *versionNotifier) gameChanged(game *boardgame.Game) {
	v.notifyVersion <- gameVersionChanged{
		Id:      game.Id(),
		Version: game.Version(),
	}
}

func (v *versionNotifier) done() {
	close(v.doneChan)
}

func (v *versionNotifier) workLoop() {
	for {
		select {
		case s := <-v.register:
			v.registerSocket(s)
		case s := <-v.unregister:
			v.unregisterSocket(s)
		case rec := <-v.notifyVersion:
			debugLog("Sending message for " + rec.Id + " " + strconv.Itoa(rec.Version))
			//Send message
			bucket, ok := v.sockets[rec.Id]
			if ok {
				//Someone's listening!
				for socket := range bucket {
					socket.send <- []byte(strconv.Itoa(rec.Version))
				}
			}
		case <-v.doneChan:
			break
		}
	}
}

func debugLog(message string) {
	if !debugSockets {
		return
	}

	log.Println(message)
}

func (v *versionNotifier) registerSocket(s *socket) {
	//Should only be called by workLoop

	debugLog("Socket registering")

	bucket, ok := v.sockets[s.gameId]

	if !ok {
		bucket = make(map[*socket]bool)
		v.sockets[s.gameId] = bucket
	}

	bucket[s] = true
}

func (v *versionNotifier) unregisterSocket(s *socket) {
	//Should only be called by workloop

	debugLog("Socket unregistering")

	bucket, ok := v.sockets[s.gameId]

	if !ok {
		return
	}

	delete(bucket, s)
}
<|endoftext|>


<commit_before><commit_msg>Cleanup docs.<commit_after><|endoftext|>


<commit_before>// Main app

package main

import (
	"fmt"
	"log"
	"net/http"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there")
	log.Println("Log: handler request")
}

func main() {
	log.Println("Log: main app is running")
	http.HandleFunc("/", indexHandler)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Make multiple handlers<commit_after>// Main app

package main

import (
	"fmt"
	"log"
	"net/http"
)

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hi there, I'm Will.")
	log.Println("Log: handler request")
}

func helloHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Hello again")
	log.Println("Log: handler request")
}

func main() {
	log.Println("Log: main app is running")
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/hello/", helloHandler)
	err := http.ListenAndServe(":8080", nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}
)

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, lockFile string) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	m := model.NewUser(ctx)
	var (
		sqlStr    string
		installer func(string) error
		ok        bool
	)
	adminUser := ctx.Form(`adminUser`)
	adminPass := ctx.Form(`adminPass`)
	adminEmail := ctx.Form(`adminEmail`)
	if len(adminUser) == 0 {
		err = ctx.E(`管理员用户名不能为空`)
		return
	}
	if !com.IsUsername(adminUser) {
		err = errors.New(ctx.T(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`))
		return
	}
	if len(adminPass) < 8 {
		err = ctx.E(`管理员密码不能少于8个字符`)
		return
	}
	if len(adminEmail) == 0 {
		err = ctx.E(`管理员邮箱不能为空`)
		return
	}
	if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
		err = ctx.E(`管理员邮箱格式不正确`)
		return
	}
	err = ctx.MustBind(&config.DefaultConfig.DB)
	config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
	config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
	if config.DefaultConfig.DB.Type == `sqlite` {
		config.DefaultConfig.DB.User = ``
		config.DefaultConfig.DB.Password = ``
		config.DefaultConfig.DB.Host = ``
		if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
			config.DefaultConfig.DB.Database += `.db`
		}
	}
	if err != nil {
		return
	}
	//连接数据库
	err = config.ConnectDB(config.DefaultConfig)
	if err != nil {
		err = createDatabase(err)
	}
	if err != nil {
		return
	}
	//创建数据库数据
	installer, ok = config.DBInstallers[config.DefaultConfig.DB.Type]
	if !ok {
		err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
		return
	}
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	err = m.Register(adminUser, adminPass, adminEmail)
	if err != nil {
		return
	}
	config.DefaultConfig.InitSecretKey()

	//保存数据库账号到配置文件
	err = config.DefaultConfig.SaveToFile()
	if err != nil {
		return
	}

	//生成锁文件
	err = config.SetInstalled(lockFile)
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			err = install(ctx, sqlFile, lockFile)
			if err != nil {
				break
			}
		}
		defer func() {
			if err == nil {
				time.Sleep(1 * time.Second)
				config.DefaultCLIConfig.RunStartup()
				if err := Upgrade(); err != nil {
					log.Error(err)
				}
			}
		}()
		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		} else {
			if err != nil {
				goto DIE
			}
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package setup

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"

	"github.com/admpub/log"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	"github.com/admpub/nging/application/model"
)

type ProgressInfo struct {
	Finished  int64
	TotalSize int64
	Summary   string
	Timestamp int64
}

var (
	installProgress   *ProgressInfo
	installedProgress = &ProgressInfo{
		Finished:  1,
		TotalSize: 1,
	}
	uninstallProgress = &ProgressInfo{
		Finished:  0,
		TotalSize: 1,
	}

	OnInstalled func(ctx context.Context) error
)

func init() {
	handler.Register(func(e echo.RouteRegister) {
		e.Route("GET,POST", `/setup`, Setup)
		e.Route("GET", `/progress`, Progress)
		e.Route("GET,POST", `/license`, License)
	})
}

func Progress(ctx echo.Context) error {
	data := ctx.Data()
	if config.IsInstalled() {
		data.SetInfo(ctx.T(`已经安装过了`), 0)
		data.SetData(installedProgress)
	} else {
		if installProgress == nil {
			data.SetInfo(ctx.T(`尚未开始`), 1)
			data.SetData(uninstallProgress)
		} else {
			data.SetInfo(ctx.T(`安装中`), 1)
			data.SetData(installProgress)
		}
	}
	return ctx.JSON(data)
}

func install(ctx echo.Context, sqlFile string, lockFile string) (err error) {
	installProgress = &ProgressInfo{
		Timestamp: time.Now().Local().Unix(),
	}
	defer func() {
		if err != nil {
			installProgress = nil
		}
	}()
	m := model.NewUser(ctx)
	var (
		sqlStr    string
		installer func(string) error
		ok        bool
	)
	adminUser := ctx.Form(`adminUser`)
	adminPass := ctx.Form(`adminPass`)
	adminEmail := ctx.Form(`adminEmail`)
	if len(adminUser) == 0 {
		err = ctx.E(`管理员用户名不能为空`)
		return
	}
	if !com.IsUsername(adminUser) {
		err = errors.New(ctx.T(`管理员名不能包含特殊字符(只能由字母、数字、下划线和汉字组成)`))
		return
	}
	if len(adminPass) < 8 {
		err = ctx.E(`管理员密码不能少于8个字符`)
		return
	}
	if len(adminEmail) == 0 {
		err = ctx.E(`管理员邮箱不能为空`)
		return
	}
	if !ctx.Validate(`adminEmail`, adminEmail, `email`).Ok() {
		err = ctx.E(`管理员邮箱格式不正确`)
		return
	}
	err = ctx.MustBind(&config.DefaultConfig.DB)
	config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "'", "", -1)
	config.DefaultConfig.DB.Database = strings.Replace(config.DefaultConfig.DB.Database, "`", "", -1)
	if config.DefaultConfig.DB.Type == `sqlite` {
		config.DefaultConfig.DB.User = ``
		config.DefaultConfig.DB.Password = ``
		config.DefaultConfig.DB.Host = ``
		if strings.HasSuffix(config.DefaultConfig.DB.Database, `.db`) == false {
			config.DefaultConfig.DB.Database += `.db`
		}
	}
	if err != nil {
		return
	}
	//连接数据库
	err = config.ConnectDB(config.DefaultConfig)
	if err != nil {
		err = createDatabase(err)
	}
	if err != nil {
		return
	}
	//创建数据库数据
	installer, ok = config.DBInstallers[config.DefaultConfig.DB.Type]
	if !ok {
		err = ctx.E(`不支持安装到%s`, config.DefaultConfig.DB.Type)
		return
	}
	installProgress.TotalSize, err = com.FileSize(sqlFile)
	if err != nil {
		return
	}
	installProgress.TotalSize += int64(len(handler.OfficialSQL))
	installFunction := func(line string) (rErr error) {
		installProgress.Finished += int64(len(line)) + 1
		if strings.HasPrefix(line, `--`) {
			return nil
		}
		if strings.HasPrefix(line, `/*`) && strings.HasSuffix(line, `*/;`) {
			return nil
		}
		line = strings.TrimSpace(line)
		sqlStr += line
		if strings.HasSuffix(line, `;`) && len(sqlStr) > 0 {
			//installProgress.Summary = sqlStr
			defer func() {
				sqlStr = ``
			}()
			return installer(sqlStr)
		}
		return nil
	}
	err = com.SeekFileLines(sqlFile, installFunction)
	if err != nil {
		return
	}
	for _, line := range strings.Split(handler.OfficialSQL, "\n") {
		err = installFunction(line)
		if err != nil {
			return
		}
	}
	err = m.Register(adminUser, adminPass, adminEmail)
	if err != nil {
		return
	}
	config.DefaultConfig.InitSecretKey()

	//保存数据库账号到配置文件
	err = config.DefaultConfig.SaveToFile()
	if err != nil {
		return
	}

	//生成锁文件
	err = config.SetInstalled(lockFile)
	if err == nil && OnInstalled != nil {
		err = OnInstalled(ctx)
	}
	return
}

func Setup(ctx echo.Context) error {
	var err error
	lockFile := filepath.Join(echo.Wd(), `installed.lock`)
	if info, err := os.Stat(lockFile); err == nil && info.IsDir() == false {
		msg := ctx.T(`已经安装过了。如要重新安装，请先删除%s`, lockFile)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	sqlFiles, err := config.GetSQLInstallFiles()
	if err != nil {
		msg := ctx.T(`找不到文件%s，无法安装`, `config/install.sql`)
		if ctx.IsAjax() {
			return ctx.JSON(ctx.Data().SetInfo(msg, 0))
		}
		return ctx.String(msg)
	}
	if ctx.IsPost() && installProgress == nil {
		data := ctx.Data()
		for _, sqlFile := range sqlFiles {
			err = install(ctx, sqlFile, lockFile)
			if err != nil {
				break
			}
		}
		defer func() {
			if err == nil {
				time.Sleep(1 * time.Second)
				config.DefaultCLIConfig.RunStartup()
				if err := Upgrade(); err != nil {
					log.Error(err)
				}
			}
		}()
		if ctx.IsAjax() {
			if err != nil {
				data.SetError(err)
			} else {
				data.SetInfo(ctx.T(`安装成功`)).SetData(installProgress)
			}
			return ctx.JSON(data)
		} else {
			if err != nil {
				goto DIE
			}
		}
		handler.SendOk(ctx, ctx.T(`安装成功`))
		return ctx.Redirect(handler.URLFor(`/`))
	}

DIE:
	ctx.Set(`dbEngines`, config.DBEngines.Slice())
	return ctx.Render(`setup`, handler.Err(ctx, err))
}

func createDatabase(err error) error {
	if fn, ok := config.DBCreaters[config.DefaultConfig.DB.Type]; ok {
		return fn(err, config.DefaultConfig)
	}
	return err
}
<|endoftext|>