<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	_ "github.com/mattn/go-sqlite3"

	"github.com/Masterminds/squirrel"
	"github.com/jmoiron/sqlx"
	"gopkg.in/alecthomas/kingpin.v2"
)

// CountBuilder is a squirrel select builder whose columns match Count fields.
var CountBuilder = squirrel.Select().
	Column(squirrel.Alias(squirrel.Expr("CASE WHEN rname IS NULL THEN \"\" ELSE rname END"), "rname")).
	Column(squirrel.Alias(squirrel.Expr("CASE WHEN strand IS NULL THEN 0 ELSE strand END"), "strand")).
	Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
	Column(squirrel.Alias(squirrel.Expr("TOTAL(copy_number)"), "copyNum"))

// Count is a databases row with record count information.
type Count struct {
	Chrom   string `db:"rname"`
	Ori     int    `db:"strand"`
	Count   int    `db:"count"`
	CopyNum int    `db:"copyNum"`
}

const prog = "htsdb-count-reads"
const version = "0.3"
const descr = `Print the number of reads and read copies stored in the
database. Supports grouping by reference, orientation or both. Provided SQL
filter will apply to all counts.`

var (
	app = kingpin.New(prog, descr)

	dbFile = app.Flag("db", "File to SQLite database.").
		PlaceHolder("<file>").Required().String()
	tab = app.Flag("table", "Database table name.").
		Default("sample").String()
	where = app.Flag("where", "SQL filter to inject in WHERE clause.").
		PlaceHolder("<SQL>").String()
	as = app.Flag("as", "Name to print describing the count/s.").
		Default("all").String()
	header = app.Flag("header", "Print header line.").
		Bool()
	groupByChrom = app.Flag("by-ref", "Group counts by reference.").
			Bool()
	groupByOri = app.Flag("by-ori", "Group counts by orientation.").
			Bool()
)

func main() {
	// read command line args and options
	app.HelpFlag.Short('h')
	app.Version(version)
	_, err := app.Parse(os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%s", err)
	}

	// assemble sqlx select builders
	countBuilder := CountBuilder.From(*tab)
	if *where != "" {
		countBuilder = countBuilder.Where(*where)
	}
	if *groupByChrom == true {
		countBuilder = countBuilder.GroupBy("rname")
	}
	if *groupByOri == true {
		countBuilder = countBuilder.GroupBy("strand")
	}

	// open database connections.
	var db *sqlx.DB
	if db, err = sqlx.Connect("sqlite3", *dbFile); err != nil {
		log.Fatal(err)
	}

	// prepare statements.
	query, _, err := countBuilder.ToSql()
	if err != nil {
		log.Fatal(err)
	}

	// get the count
	var counts []Count
	if err = db.Select(&counts, query); err != nil {
		log.Fatal(err)
	}

	// print results.
	if *groupByChrom == true && *groupByOri == true {
		if *header == true {
			fmt.Printf("category\tref\tori\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%s\t%d\t%d\t%d\n", *as, c.Chrom, c.Ori, c.Count, c.CopyNum)
		}
	} else if *groupByChrom == true {
		if *header == true {
			fmt.Printf("category\tref\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%s\t%d\t%d\n", *as, c.Chrom, c.Count, c.CopyNum)
		}
	} else if *groupByOri == true {
		if *header == true {
			fmt.Printf("category\tori\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%d\t%d\t%d\n", *as, c.Ori, c.Count, c.CopyNum)
		}
	} else {
		if *header == true {
			fmt.Printf("category\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%d\t%d\n", *as, c.Count, c.CopyNum)
		}
	}
}
<commit_msg>htsdb-count-reads: fix query for case when no reads are found<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	_ "github.com/mattn/go-sqlite3"

	"github.com/Masterminds/squirrel"
	"github.com/jmoiron/sqlx"
	"gopkg.in/alecthomas/kingpin.v2"
)

// CountBuilder is a squirrel select builder whose columns match Count fields.
var CountBuilder = squirrel.Select().
	Column(squirrel.Alias(squirrel.Expr("CASE WHEN rname IS NULL THEN \"\" ELSE rname END"), "rname")).
	Column(squirrel.Alias(squirrel.Expr("CASE WHEN strand IS NULL THEN 0 ELSE strand END"), "strand")).
	Column(squirrel.Alias(squirrel.Expr("COUNT(*)"), "count")).
	Column(squirrel.Alias(squirrel.Expr("CAST(TOTAL(copy_number) AS INTEGER)"), "copyNum"))

// Count is a databases row with record count information.
type Count struct {
	Chrom   string `db:"rname"`
	Ori     int    `db:"strand"`
	Count   int    `db:"count"`
	CopyNum int    `db:"copyNum"`
}

const prog = "htsdb-count-reads"
const version = "0.4"
const descr = `Print the number of reads and read copies stored in the
database. Supports grouping by reference, orientation or both. Provided SQL
filter will apply to all counts.`

var (
	app = kingpin.New(prog, descr)

	dbFile = app.Flag("db", "File to SQLite database.").
		PlaceHolder("<file>").Required().String()
	tab = app.Flag("table", "Database table name.").
		Default("sample").String()
	where = app.Flag("where", "SQL filter to inject in WHERE clause.").
		PlaceHolder("<SQL>").String()
	as = app.Flag("as", "Name to print describing the count/s.").
		Default("all").String()
	header = app.Flag("header", "Print header line.").
		Bool()
	groupByChrom = app.Flag("by-ref", "Group counts by reference.").
			Bool()
	groupByOri = app.Flag("by-ori", "Group counts by orientation.").
			Bool()
)

func main() {
	// read command line args and options
	app.HelpFlag.Short('h')
	app.Version(version)
	_, err := app.Parse(os.Args[1:])
	if err != nil {
		kingpin.Fatalf("%s", err)
	}

	// assemble sqlx select builders
	countBuilder := CountBuilder.From(*tab)
	if *where != "" {
		countBuilder = countBuilder.Where(*where)
	}
	if *groupByChrom == true {
		countBuilder = countBuilder.GroupBy("rname")
	}
	if *groupByOri == true {
		countBuilder = countBuilder.GroupBy("strand")
	}

	// open database connections.
	var db *sqlx.DB
	if db, err = sqlx.Connect("sqlite3", *dbFile); err != nil {
		log.Fatal(err)
	}

	// prepare statements.
	query, _, err := countBuilder.ToSql()
	if err != nil {
		log.Fatal(err)
	}

	// get the count
	var counts []Count
	if err = db.Select(&counts, query); err != nil {
		log.Fatal(err)
	}

	// print results.
	if *groupByChrom == true && *groupByOri == true {
		if *header == true {
			fmt.Printf("category\tref\tori\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%s\t%d\t%d\t%d\n", *as, c.Chrom, c.Ori, c.Count, c.CopyNum)
		}
	} else if *groupByChrom == true {
		if *header == true {
			fmt.Printf("category\tref\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%s\t%d\t%d\n", *as, c.Chrom, c.Count, c.CopyNum)
		}
	} else if *groupByOri == true {
		if *header == true {
			fmt.Printf("category\tori\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%d\t%d\t%d\n", *as, c.Ori, c.Count, c.CopyNum)
		}
	} else {
		if *header == true {
			fmt.Printf("category\tcount\tcopyNumber\n")
		}
		for _, c := range counts {
			fmt.Printf("%s\t%d\t%d\n", *as, c.Count, c.CopyNum)
		}
	}
}
<|endoftext|>


<commit_before>package openapi

// Document represents a OpenAPI Specification document.
type Document struct {
	Version      string `yaml:"openapi"`
	Info         Info
	Servers      []Server
	Paths        Paths
	Components   Components
	Security     SecurityRequirement
	Tags         []Tag
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
}

// Info Object
type Info struct {
	Title          string
	Description    string
	TermsOfService string `yaml:"termsOfService"`
	Contact        Contact
	License        License
	Version        string
}

// Contact Object
type Contact struct {
	Name  string
	URL   string
	Email string
}

// License Object
type License struct {
	Name string
	URL  string
}

// Server Object
type Server struct {
	URL         string
	Description string
	Variables   map[string]*ServerVariable
}

// ServerVariable Object
type ServerVariable struct {
	Enum        []string
	Default     string
	Description string
}

// Paths Object
type Paths map[string]PathItem

// PathItem Object
type PathItem struct {
	Ref         string
	Summary     string
	Description string
	Get         Operation
	Put         Operation
	Post        Operation
	Delete      Operation
	Options     Operation
	Head        Operation
	Patch       Operation
	Trace       Operation
	Servers     []Server
	Parameters  Parameter
}

// Operation Object
type Operation struct {
	Tags         []string
	Summary      string
	Description  string
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
	OperationID  string                `yaml:"operationId"`
	Parameters   []Parameter
	RequestBody  RequestBody `yaml:"requestBody"`
	Responses    Responses
	Callbacks    map[string]Callback
	Deprecated   bool
	Security     SecurityRequirement
	Servers      []*Server
}

// Parameter Object
type Parameter struct {
	Name            string
	In              string
	Description     string
	Required        bool
	Deprecated      string
	AllowEmptyValue bool `yaml:"allowEmptyValue"`

	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
	Schema        Schema
	Example       interface{}
	Examples      map[string]Example

	Content map[string]MediaType

	Ref string `yaml:"$ref"`
}

// RequestBody Object
type RequestBody struct {
	Description string
	Content     map[string]MediaType
	Required    bool

	Ref string `yaml:"$ref"`
}

// Responses Object
type Responses map[string]*Response

// Response Object
type Response struct {
	Description string
	Headers     map[string]Header
	Content     map[string]MediaType
	Links       map[string]Link

	Ref string `yaml:"$ref"`
}

// Callbacks Object
type Callbacks map[string]Callback

// Callback Object
type Callback map[string]PathItem

// Schema Object
type Schema struct {
	Title            string
	MultipleOf       int `yaml:"multipleOf"`
	Maximum          int
	ExclusiveMaximum bool `yaml:"exclusiveMaximum"`
	Minimum          int
	ExclusiveMinimum bool `yaml:"exclusiveMinimum"`
	MaxLength        int  `yaml:"maxLength"`
	MinLength        int  `yaml:"minLength"`
	Pattern          string
	MaxItems         int `yaml:"maxItems"`
	MinItems         int `yaml:"minItems"`
	MaxProperties    int `yaml:"maxProperties"`
	MinProperties    int `yaml:"minProperties"`
	Required         bool
	Enum             []string

	Type                       string
	AllOf                      *Schema `yaml:"allOf"`
	OneOf                      *Schema `yaml:"oneOf"`
	AnyOf                      *Schema `yaml:"anyOf"`
	Not                        *Schema
	Items                      []*Schema
	Properties                 map[string]*Schema
	EnableAdditionalProperties bool `yaml:"additionalProperties"`
	Description                string
	Format                     string
	Default                    string

	Nullable      bool
	Discriminator Discriminator
	ReadOnly      bool `yaml:"readOnly"`
	WriteOnly     bool `yaml:"writeOnly"`
	XML           XML
	ExternalDocs  ExternalDocumentation `yaml:"externalDocs"`
	Example       interface{}
	Deprecated    bool

	Ref string `yaml:"$ref"`
}

// Example Object
type Example struct {
	Summary       string
	Description   string
	Value         interface{}
	ExternalValue interface{} `yaml:"externalValue"`

	Ref string `yaml:"$ref"`
}

// MediaType Object
type MediaType struct {
	Schema   Schema
	Example  interface{}
	Examples map[string]Example
	Encoding map[string]Encoding

	Ref string `yaml:"$ref"`
}

// Header Object
type Header struct {
	Description     string
	Required        bool
	Deprecated      string
	AllowEmptyValue bool `yaml:"allowEmptyValue"`

	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
	Schema        Schema
	Example       interface{}
	Examples      map[string]Example

	Content map[string]MediaType

	Ref string `yaml:"$ref"`
}

// Link Object
type Link struct {
	OperationRef string `yaml:"operationRef"`
	OperationID  string `yaml:"operationId"`
	Parameters   map[string]interface{}
	RequestBody  interface{} `yaml:"requestBody"`
	Description  string
	Server       Server

	Ref string `yaml:"$ref"`
}

// Encoding Object
type Encoding struct {
	ContentType   string `yaml:"contentType"`
	Headers       map[string]Header
	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
}

// Discriminator Object
type Discriminator struct {
	PropertyName string `yaml:"propertyName"`
	Mapping      map[string]string
}

// XML Object
type XML struct {
	Name      string
	Namespace string
	Prefix    string
	Attribute bool
	Wrapped   bool
}

// Components Object
type Components struct {
	Schemas         map[string]Schema
	Responses       map[string]Response
	Parameters      map[string]Parameter
	Examples        map[string]Example
	RequestBodies   map[string]RequestBody `yaml:"requestBodies"`
	Headers         map[string]Header
	SecuritySchemes map[string]SecurityScheme `yaml:"securitySchemes"`
	Links           map[string]Link
	Callbacks       map[string]Callback
}

// SecurityScheme Object
type SecurityScheme struct {
	Type             string
	Description      string
	Name             string
	In               string
	Scheme           string
	BearerFormat     string `yaml:"bearerFormat"`
	Flows            OAuthFlows
	OpenIDConnectURL string `yaml:"openIdConnectUrl"`

	Ref string `yaml:"$ref"`
}

// OAuthFlows Object
type OAuthFlows struct {
	Implicit          OAuthFlow
	Password          OAuthFlow
	ClientCredentials OAuthFlow `yaml:"clientCredentials"`
	AuthorizationCode OAuthFlow `yaml:"authorizationCode"`
}

// OAuthFlow Object
type OAuthFlow struct {
	AuthorizationURL string `yaml:"authorizationUrl"`
	TokenURL         string `yaml:"tokenUrl"`
	RefreshURL       string `yaml:"refreshUrl"`
	Scopes           map[string]string
}

// SecurityRequirement Object
type SecurityRequirement map[string][]string

// Tag Object
type Tag struct {
	Name         string
	Description  string
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
}

// ExternalDocumentation Object
type ExternalDocumentation struct {
	Description string
	URL         string
}
<commit_msg>some bug fix<commit_after>package openapi

// Document represents a OpenAPI Specification document.
type Document struct {
	Version      string `yaml:"openapi"`
	Info         Info
	Servers      []Server
	Paths        Paths
	Components   Components
	Security     SecurityRequirement
	Tags         []Tag
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
}

// Info Object
type Info struct {
	Title          string
	Description    string
	TermsOfService string `yaml:"termsOfService"`
	Contact        Contact
	License        License
	Version        string
}

// Contact Object
type Contact struct {
	Name  string
	URL   string
	Email string
}

// License Object
type License struct {
	Name string
	URL  string
}

// Server Object
type Server struct {
	URL         string
	Description string
	Variables   map[string]*ServerVariable
}

// ServerVariable Object
type ServerVariable struct {
	Enum        []string
	Default     string
	Description string
}

// Paths Object
type Paths map[string]PathItem

// PathItem Object
type PathItem struct {
	Ref         string
	Summary     string
	Description string
	Get         Operation
	Put         Operation
	Post        Operation
	Delete      Operation
	Options     Operation
	Head        Operation
	Patch       Operation
	Trace       Operation
	Servers     []Server
	Parameters  []Parameter
}

// Operation Object
type Operation struct {
	Tags         []string
	Summary      string
	Description  string
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
	OperationID  string                `yaml:"operationId"`
	Parameters   []Parameter
	RequestBody  RequestBody `yaml:"requestBody"`
	Responses    Responses
	Callbacks    map[string]Callback
	Deprecated   bool
	Security     SecurityRequirement
	Servers      []*Server
}

// Parameter Object
type Parameter struct {
	Name            string
	In              string
	Description     string
	Required        bool
	Deprecated      string
	AllowEmptyValue bool `yaml:"allowEmptyValue"`

	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
	Schema        Schema
	Example       interface{}
	Examples      map[string]Example

	Content map[string]MediaType

	Ref string `yaml:"$ref"`
}

// RequestBody Object
type RequestBody struct {
	Description string
	Content     map[string]MediaType
	Required    bool

	Ref string `yaml:"$ref"`
}

// Responses Object
type Responses map[string]*Response

// Response Object
type Response struct {
	Description string
	Headers     map[string]Header
	Content     map[string]MediaType
	Links       map[string]Link

	Ref string `yaml:"$ref"`
}

// Callbacks Object
type Callbacks map[string]Callback

// Callback Object
type Callback map[string]PathItem

// Schema Object
type Schema struct {
	Title            string
	MultipleOf       int `yaml:"multipleOf"`
	Maximum          int
	ExclusiveMaximum bool `yaml:"exclusiveMaximum"`
	Minimum          int
	ExclusiveMinimum bool `yaml:"exclusiveMinimum"`
	MaxLength        int  `yaml:"maxLength"`
	MinLength        int  `yaml:"minLength"`
	Pattern          string
	MaxItems         int `yaml:"maxItems"`
	MinItems         int `yaml:"minItems"`
	MaxProperties    int `yaml:"maxProperties"`
	MinProperties    int `yaml:"minProperties"`
	Required         bool
	Enum             []string

	Type                       string
	AllOf                      *Schema `yaml:"allOf"`
	OneOf                      *Schema `yaml:"oneOf"`
	AnyOf                      *Schema `yaml:"anyOf"`
	Not                        *Schema
	Items                      *Schema
	Properties                 map[string]*Schema
	EnableAdditionalProperties bool `yaml:"additionalProperties"`
	Description                string
	Format                     string
	Default                    string

	Nullable      bool
	Discriminator Discriminator
	ReadOnly      bool `yaml:"readOnly"`
	WriteOnly     bool `yaml:"writeOnly"`
	XML           XML
	ExternalDocs  ExternalDocumentation `yaml:"externalDocs"`
	Example       interface{}
	Deprecated    bool

	Ref string `yaml:"$ref"`
}

// Example Object
type Example struct {
	Summary       string
	Description   string
	Value         interface{}
	ExternalValue interface{} `yaml:"externalValue"`

	Ref string `yaml:"$ref"`
}

// MediaType Object
type MediaType struct {
	Schema   Schema
	Example  interface{}
	Examples map[string]Example
	Encoding map[string]Encoding

	Ref string `yaml:"$ref"`
}

// Header Object
type Header struct {
	Description     string
	Required        bool
	Deprecated      string
	AllowEmptyValue bool `yaml:"allowEmptyValue"`

	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
	Schema        Schema
	Example       interface{}
	Examples      map[string]Example

	Content map[string]MediaType

	Ref string `yaml:"$ref"`
}

// Link Object
type Link struct {
	OperationRef string `yaml:"operationRef"`
	OperationID  string `yaml:"operationId"`
	Parameters   map[string]interface{}
	RequestBody  interface{} `yaml:"requestBody"`
	Description  string
	Server       Server

	Ref string `yaml:"$ref"`
}

// Encoding Object
type Encoding struct {
	ContentType   string `yaml:"contentType"`
	Headers       map[string]Header
	Style         string
	Explode       bool
	AllowReserved bool `yaml:"allowReserved"`
}

// Discriminator Object
type Discriminator struct {
	PropertyName string `yaml:"propertyName"`
	Mapping      map[string]string
}

// XML Object
type XML struct {
	Name      string
	Namespace string
	Prefix    string
	Attribute bool
	Wrapped   bool
}

// Components Object
type Components struct {
	Schemas         map[string]Schema
	Responses       map[string]Response
	Parameters      map[string]Parameter
	Examples        map[string]Example
	RequestBodies   map[string]RequestBody `yaml:"requestBodies"`
	Headers         map[string]Header
	SecuritySchemes map[string]SecurityScheme `yaml:"securitySchemes"`
	Links           map[string]Link
	Callbacks       map[string]Callback
}

// SecurityScheme Object
type SecurityScheme struct {
	Type             string
	Description      string
	Name             string
	In               string
	Scheme           string
	BearerFormat     string `yaml:"bearerFormat"`
	Flows            OAuthFlows
	OpenIDConnectURL string `yaml:"openIdConnectUrl"`

	Ref string `yaml:"$ref"`
}

// OAuthFlows Object
type OAuthFlows struct {
	Implicit          OAuthFlow
	Password          OAuthFlow
	ClientCredentials OAuthFlow `yaml:"clientCredentials"`
	AuthorizationCode OAuthFlow `yaml:"authorizationCode"`
}

// OAuthFlow Object
type OAuthFlow struct {
	AuthorizationURL string `yaml:"authorizationUrl"`
	TokenURL         string `yaml:"tokenUrl"`
	RefreshURL       string `yaml:"refreshUrl"`
	Scopes           map[string]string
}

// SecurityRequirement Object
type SecurityRequirement []map[string][]string

// Tag Object
type Tag struct {
	Name         string
	Description  string
	ExternalDocs ExternalDocumentation `yaml:"externalDocs"`
}

// ExternalDocumentation Object
type ExternalDocumentation struct {
	Description string
	URL         string
}
<|endoftext|>


<commit_before>package weed_server

import (
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const BufferSizeLimit = 1024 * 1024 * 2

// VolumeCopy copy the .idx .dat .vif files, and mount the volume
func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v != nil {

		glog.V(0).Infof("volume %d already exists. deleted before copying...", req.VolumeId)

		err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId))
		if err != nil {
			return nil, fmt.Errorf("failed to mount existing volume %d: %v", req.VolumeId, err)
		}

		err = vs.store.DeleteVolume(needle.VolumeId(req.VolumeId))
		if err != nil {
			return nil, fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err)
		}

		glog.V(0).Infof("deleted exisitng volume %d before copying.", req.VolumeId)
	}

	location := vs.store.FindFreeLocation()
	if location == nil {
		return nil, fmt.Errorf("no space left")
	}

	// the master will not start compaction for read-only volumes, so it is safe to just copy files directly
	// copy .dat and .idx files
	//   read .idx .dat file size and timestamp
	//   send .idx file
	//   send .dat file
	//   confirm size and timestamp
	var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse
	var volumeFileName, idxFileName, datFileName string
	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		var err error
		volFileInfoResp, err = client.ReadVolumeFileStatus(context.Background(),
			&volume_server_pb.ReadVolumeFileStatusRequest{
				VolumeId: req.VolumeId,
			})
		if nil != err {
			return fmt.Errorf("read volume file status failed, %v", err)
		}

		volumeFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId))

		// println("source:", volFileInfoResp.String())
		// copy ecx file
		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, volumeFileName, ".idx", false, false); err != nil {
			return err
		}

		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil {
			return err
		}

		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".vif", false, true); err != nil {
			return err
		}

		return nil
	})

	idxFileName = volumeFileName + ".idx"
	datFileName = volumeFileName + ".dat"

	defer func() {
		if err != nil && volumeFileName != "" {
			os.Remove(idxFileName)
			os.Remove(datFileName)
			os.Remove(volumeFileName + ".vif")
		}
	}()

	if err != nil && volumeFileName != "" {
		return nil, err
	}

	if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16
		return nil, err
	}

	// mount the volume
	err = vs.store.MountVolume(needle.VolumeId(req.VolumeId))
	if err != nil {
		return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
	}

	return &volume_server_pb.VolumeCopyResponse{
		LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second),
	}, err
}

func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool) error {

	copyFileClient, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
		VolumeId:                 vid,
		Ext:                      ext,
		CompactionRevision:       compactRevision,
		StopOffset:               stopOffset,
		Collection:               collection,
		IsEcVolume:               isEcVolume,
		IgnoreSourceFileNotFound: ignoreSourceFileNotFound,
	})
	if err != nil {
		return fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err)
	}

	err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend)
	if err != nil {
		return fmt.Errorf("failed to copy %s file: %v", baseFileName+ext, err)
	}

	return nil

}

/**
only check the the differ of the file size
todo: maybe should check the received count and deleted count of the volume
*/
func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse, idxFileName, datFileName string) error {
	stat, err := os.Stat(idxFileName)
	if err != nil {
		return fmt.Errorf("stat idx file %s failed, %v", idxFileName, err)
	}
	if originFileInf.IdxFileSize != uint64(stat.Size()) {
		return fmt.Errorf("idx file %s size [%v] is not same as origin file size [%v]",
			idxFileName, stat.Size(), originFileInf.IdxFileSize)
	}

	stat, err = os.Stat(datFileName)
	if err != nil {
		return fmt.Errorf("get dat file info failed, %v", err)
	}
	if originFileInf.DatFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the dat file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.DatFileSize)
	}
	return nil
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool) error {
	glog.V(4).Infof("writing to %s", fileName)
	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	if isAppend {
		flags = os.O_WRONLY | os.O_CREATE
	}
	dst, err := os.OpenFile(fileName, flags, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
		wt.MaybeSlowdown(int64(len(resp.FileContent)))
	}
	return nil
}

func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
	resp := &volume_server_pb.ReadVolumeFileStatusResponse{}
	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	resp.VolumeId = req.VolumeId
	datSize, idxSize, modTime := v.FileStat()
	resp.DatFileSize = datSize
	resp.IdxFileSize = idxSize
	resp.DatFileTimestampSeconds = uint64(modTime.Unix())
	resp.IdxFileTimestampSeconds = uint64(modTime.Unix())
	resp.FileCount = v.FileCount()
	resp.CompactionRevision = uint32(v.CompactionRevision)
	resp.Collection = v.Collection
	return resp, nil
}

// CopyFile client pulls the volume related file from the source server.
// if req.CompactionRevision != math.MaxUint32, it ensures the compact revision is as expected
// The copying still stop at req.StopOffset, but you can set it to math.MaxUint64 in order to read all data.
func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {

	var fileName string
	if !req.IsEcVolume {
		v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
		if v == nil {
			return fmt.Errorf("not found volume id %d", req.VolumeId)
		}

		if uint32(v.CompactionRevision) != req.CompactionRevision && req.CompactionRevision != math.MaxUint32 {
			return fmt.Errorf("volume %d is compacted", req.VolumeId)
		}
		fileName = v.FileName() + req.Ext
	} else {
		baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) + req.Ext
		for _, location := range vs.store.Locations {
			tName := util.Join(location.Directory, baseFileName)
			if util.FileExists(tName) {
				fileName = tName
			}
		}
		if fileName == "" {
			if req.IgnoreSourceFileNotFound {
				return nil
			}
			return fmt.Errorf("CopyFile not found ec volume id %d", req.VolumeId)
		}
	}

	bytesToRead := int64(req.StopOffset)

	file, err := os.Open(fileName)
	if err != nil {
		if req.IgnoreSourceFileNotFound && err == os.ErrNotExist {
			return nil
		}
		return err
	}
	defer file.Close()

	buffer := make([]byte, BufferSizeLimit)

	for bytesToRead > 0 {
		bytesread, err := file.Read(buffer)

		// println(fileName, "read", bytesread, "bytes, with target", bytesToRead)

		if err != nil {
			if err != io.EOF {
				return err
			}
			// println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
			break
		}

		if int64(bytesread) > bytesToRead {
			bytesread = int(bytesToRead)
		}
		err = stream.Send(&volume_server_pb.CopyFileResponse{
			FileContent: buffer[:bytesread],
		})
		if err != nil {
			// println("sending", bytesread, "bytes err", err.Error())
			return err
		}

		bytesToRead -= int64(bytesread)

	}

	return nil
}
<commit_msg>add error to avoid copying not found volume<commit_after>package weed_server

import (
	"context"
	"fmt"
	"io"
	"math"
	"os"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const BufferSizeLimit = 1024 * 1024 * 2

// VolumeCopy copy the .idx .dat .vif files, and mount the volume
func (vs *VolumeServer) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest) (*volume_server_pb.VolumeCopyResponse, error) {

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v != nil {

		glog.V(0).Infof("volume %d already exists. deleted before copying...", req.VolumeId)

		err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId))
		if err != nil {
			return nil, fmt.Errorf("failed to mount existing volume %d: %v", req.VolumeId, err)
		}

		err = vs.store.DeleteVolume(needle.VolumeId(req.VolumeId))
		if err != nil {
			return nil, fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err)
		}

		glog.V(0).Infof("deleted exisitng volume %d before copying.", req.VolumeId)
	}

	location := vs.store.FindFreeLocation()
	if location == nil {
		return nil, fmt.Errorf("no space left")
	}

	// the master will not start compaction for read-only volumes, so it is safe to just copy files directly
	// copy .dat and .idx files
	//   read .idx .dat file size and timestamp
	//   send .idx file
	//   send .dat file
	//   confirm size and timestamp
	var volFileInfoResp *volume_server_pb.ReadVolumeFileStatusResponse
	var volumeFileName, idxFileName, datFileName string
	err := operation.WithVolumeServerClient(req.SourceDataNode, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
		var err error
		volFileInfoResp, err = client.ReadVolumeFileStatus(context.Background(),
			&volume_server_pb.ReadVolumeFileStatusRequest{
				VolumeId: req.VolumeId,
			})
		if nil != err {
			return fmt.Errorf("read volume file status failed, %v", err)
		}

		volumeFileName = storage.VolumeFileName(location.Directory, volFileInfoResp.Collection, int(req.VolumeId))

		// println("source:", volFileInfoResp.String())
		// copy ecx file
		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.IdxFileSize, volumeFileName, ".idx", false, false); err != nil {
			return err
		}

		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".dat", false, true); err != nil {
			return err
		}

		if err := vs.doCopyFile(client, false, req.Collection, req.VolumeId, volFileInfoResp.CompactionRevision, volFileInfoResp.DatFileSize, volumeFileName, ".vif", false, true); err != nil {
			return err
		}

		return nil
	})

	if err != nil {
		return nil, err
	}
	if volumeFileName == "" {
		return nil, fmt.Errorf("not found volume %d file", req.VolumeId)
	}

	idxFileName = volumeFileName + ".idx"
	datFileName = volumeFileName + ".dat"

	defer func() {
		if err != nil && volumeFileName != "" {
			os.Remove(idxFileName)
			os.Remove(datFileName)
			os.Remove(volumeFileName + ".vif")
		}
	}()

	if err = checkCopyFiles(volFileInfoResp, idxFileName, datFileName); err != nil { // added by panyc16
		return nil, err
	}

	// mount the volume
	err = vs.store.MountVolume(needle.VolumeId(req.VolumeId))
	if err != nil {
		return nil, fmt.Errorf("failed to mount volume %d: %v", req.VolumeId, err)
	}

	return &volume_server_pb.VolumeCopyResponse{
		LastAppendAtNs: volFileInfoResp.DatFileTimestampSeconds * uint64(time.Second),
	}, err
}

func (vs *VolumeServer) doCopyFile(client volume_server_pb.VolumeServerClient, isEcVolume bool, collection string, vid, compactRevision uint32, stopOffset uint64, baseFileName, ext string, isAppend, ignoreSourceFileNotFound bool) error {

	copyFileClient, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
		VolumeId:                 vid,
		Ext:                      ext,
		CompactionRevision:       compactRevision,
		StopOffset:               stopOffset,
		Collection:               collection,
		IsEcVolume:               isEcVolume,
		IgnoreSourceFileNotFound: ignoreSourceFileNotFound,
	})
	if err != nil {
		return fmt.Errorf("failed to start copying volume %d %s file: %v", vid, ext, err)
	}

	err = writeToFile(copyFileClient, baseFileName+ext, util.NewWriteThrottler(vs.compactionBytePerSecond), isAppend)
	if err != nil {
		return fmt.Errorf("failed to copy %s file: %v", baseFileName+ext, err)
	}

	return nil

}

/**
only check the the differ of the file size
todo: maybe should check the received count and deleted count of the volume
*/
func checkCopyFiles(originFileInf *volume_server_pb.ReadVolumeFileStatusResponse, idxFileName, datFileName string) error {
	stat, err := os.Stat(idxFileName)
	if err != nil {
		return fmt.Errorf("stat idx file %s failed, %v", idxFileName, err)
	}
	if originFileInf.IdxFileSize != uint64(stat.Size()) {
		return fmt.Errorf("idx file %s size [%v] is not same as origin file size [%v]",
			idxFileName, stat.Size(), originFileInf.IdxFileSize)
	}

	stat, err = os.Stat(datFileName)
	if err != nil {
		return fmt.Errorf("get dat file info failed, %v", err)
	}
	if originFileInf.DatFileSize != uint64(stat.Size()) {
		return fmt.Errorf("the dat file size [%v] is not same as origin file size [%v]",
			stat.Size(), originFileInf.DatFileSize)
	}
	return nil
}

func writeToFile(client volume_server_pb.VolumeServer_CopyFileClient, fileName string, wt *util.WriteThrottler, isAppend bool) error {
	glog.V(4).Infof("writing to %s", fileName)
	flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	if isAppend {
		flags = os.O_WRONLY | os.O_CREATE
	}
	dst, err := os.OpenFile(fileName, flags, 0644)
	if err != nil {
		return nil
	}
	defer dst.Close()

	for {
		resp, receiveErr := client.Recv()
		if receiveErr == io.EOF {
			break
		}
		if receiveErr != nil {
			return fmt.Errorf("receiving %s: %v", fileName, receiveErr)
		}
		dst.Write(resp.FileContent)
		wt.MaybeSlowdown(int64(len(resp.FileContent)))
	}
	return nil
}

func (vs *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
	resp := &volume_server_pb.ReadVolumeFileStatusResponse{}
	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}

	resp.VolumeId = req.VolumeId
	datSize, idxSize, modTime := v.FileStat()
	resp.DatFileSize = datSize
	resp.IdxFileSize = idxSize
	resp.DatFileTimestampSeconds = uint64(modTime.Unix())
	resp.IdxFileTimestampSeconds = uint64(modTime.Unix())
	resp.FileCount = v.FileCount()
	resp.CompactionRevision = uint32(v.CompactionRevision)
	resp.Collection = v.Collection
	return resp, nil
}

// CopyFile client pulls the volume related file from the source server.
// if req.CompactionRevision != math.MaxUint32, it ensures the compact revision is as expected
// The copying still stop at req.StopOffset, but you can set it to math.MaxUint64 in order to read all data.
func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {

	var fileName string
	if !req.IsEcVolume {
		v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
		if v == nil {
			return fmt.Errorf("not found volume id %d", req.VolumeId)
		}

		if uint32(v.CompactionRevision) != req.CompactionRevision && req.CompactionRevision != math.MaxUint32 {
			return fmt.Errorf("volume %d is compacted", req.VolumeId)
		}
		fileName = v.FileName() + req.Ext
	} else {
		baseFileName := erasure_coding.EcShardBaseFileName(req.Collection, int(req.VolumeId)) + req.Ext
		for _, location := range vs.store.Locations {
			tName := util.Join(location.Directory, baseFileName)
			if util.FileExists(tName) {
				fileName = tName
			}
		}
		if fileName == "" {
			if req.IgnoreSourceFileNotFound {
				return nil
			}
			return fmt.Errorf("CopyFile not found ec volume id %d", req.VolumeId)
		}
	}

	bytesToRead := int64(req.StopOffset)

	file, err := os.Open(fileName)
	if err != nil {
		if req.IgnoreSourceFileNotFound && err == os.ErrNotExist {
			return nil
		}
		return err
	}
	defer file.Close()

	buffer := make([]byte, BufferSizeLimit)

	for bytesToRead > 0 {
		bytesread, err := file.Read(buffer)

		// println(fileName, "read", bytesread, "bytes, with target", bytesToRead)

		if err != nil {
			if err != io.EOF {
				return err
			}
			// println(fileName, "read", bytesread, "bytes, with target", bytesToRead, "err", err.Error())
			break
		}

		if int64(bytesread) > bytesToRead {
			bytesread = int(bytesToRead)
		}
		err = stream.Send(&volume_server_pb.CopyFileResponse{
			FileContent: buffer[:bytesread],
		})
		if err != nil {
			// println("sending", bytesread, "bytes err", err.Error())
			return err
		}

		bytesToRead -= int64(bytesread)

	}

	return nil
}
<|endoftext|>


<commit_before>package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"regexp"
	"strconv"
)

type Lang string

const (
	LangEng = "english"
	LangRus = "russian"
)

const (
	PicturePathPattern = "http://steamcommunity-a.akamaihd.net/economy/image/%s"
	InventoryEndpoint  = "http://steamcommunity.com/inventory/%d/%d/%d"
)

type ItemPict struct {
	Standard string `json:"standard"`
	Large    string `json:"large"`
}

type ItemName struct {
	Text       string `json:"text"`
	Color      string `json:"color"`
	MarketText string `json:"market_text"`
	MarketHash string `json:"market_hash"`
}

type ItemMarket struct {
	Marketable   bool `json:"marketable"`
	Restrictions int  `json:"restrictions"`
}

// Due to the JSON being string, etc... we cannot re-use EconItem
// Also, "assetid" is included as "id" not as assetid.
type InventoryItem struct {
	AppID       uint64     `json:"appid"`
	ContextID   uint64     `json:"contextid"`
	AssetID     uint64     `json:"assetid"`
	ClassID     uint64     `json:"classid"`
	InstanceID  uint64     `json:"instanceid"`
	Amount      uint64     `json:"amount"`
	Tradable    bool       `json:"tradable"`
	Currency    int        `json:"currency"`
	Pictures    ItemPict   `json:"pictures"`
	Description string     `json:"desc"`
	Name        ItemName   `json:"name"`
	Commodity   bool       `json:"commodity"`
	Type        string     `json:"type"`
	Market      ItemMarket `json:"market"`
}

type InventoryContext struct {
	ID         uint64 `json:"id,string"` /* Apparently context id needs at least 64 bits...  */
	AssetCount uint32 `json:"asset_count"`
	Name       string `json:"name"`
}

type InventoryAppStats struct {
	AppID            uint64                       `json:"appid"`
	Name             string                       `json:"name"`
	AssetCount       uint32                       `json:"asset_count"`
	Icon             string                       `json:"icon"`
	Link             string                       `json:"link"`
	InventoryLogo    string                       `json:"inventory_logo"`
	TradePermissions string                       `json:"trade_permissions"`
	Contexts         map[string]*InventoryContext `json:"rgContexts"`
}

var inventoryContextRegexp = regexp.MustCompile("var g_rgAppContextData = (.*?);")

func (session *Session) fetchInventory(sid SteamID, appID, contextID uint64, lang Lang, startAssetID uint64, items *[]InventoryItem) (hasMore bool, lastAssetID uint64, err error) {
	params := url.Values{
		"l":             {string(lang)},
		"start_assetid": {strconv.FormatUint(startAssetID, 10)},
	}

	requestURL := fmt.Sprintf(InventoryEndpoint, sid, appID, contextID)

	resp, err := session.client.Get(requestURL + "?" + params.Encode())
	if err != nil {
		return false, 0, err
	}

	type Asset struct {
		AppID      uint64 `json:"appid,string"`
		ContextID  uint64 `json:"contextid,string"`
		AssetID    uint64 `json:"assetid,string"`
		ClassID    uint64 `json:"classid,string"`
		InstanceID uint64 `json:"instanceid,string"`
		Amount     uint64 `json:"amount,string"`
	}

	type DescriptionsPart struct {
		Type  string `json:"type"`
		Value string `json:"value"`
		Color string `json:"color"`
	}

	type Description struct {
		AppID                     uint64             `json:"appid"`
		ClassID                   uint64             `json:"classid,string"`
		InstanceID                uint64             `json:"instanceid,string"`
		Currency                  int                `json:"currency"`
		BackgroundColor           string             `json:"background_color"`
		IconURL                   string             `json:"icon_url"`
		IconURLLarge              string             `json:"icon_url_large"`
		Descriptions              []DescriptionsPart `json:"descriptions"`
		Tradable                  int                `json:"tradable"`
		Name                      string             `json:"name"`
		NameColor                 string             `json:"name_color"`
		Type                      string             `json:"type"`
		MarketName                string             `json:"market_name"`
		MarketHashName            string             `json:"market_hash_name"`
		Commodity                 int                `json:"commodity"`
		MarketTradableRestriction int                `json:"market_tradable_restriction"`
		Marketable                int                `json:"marketable"`
	}

	type Response struct {
		Assets              []Asset       `json:"assets"`
		Descriptions        []Description `json:"descriptions"`
		Success             int           `json:"success"`
		HasMore             int           `json:"more_items"`
		LastAssetID         string        `json:"last_assetid"`
		TotalInventoryCount int           `json:"total_inventory_count"`
		ErrorMsg            string        `json:"error"`
	}

	var response Response
	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return false, 0, err
	}

	if response.Success == 0 {
		if len(response.ErrorMsg) != 0 {
			return false, 0, errors.New(response.ErrorMsg)
		}

		return false, 0, nil // empty inventory
	}

	descriptions := make(map[string]int)

	// Fill in descriptions map, where key
	// is "<CLASS_ID>_<INSTANCE_ID>" pattern, and
	// value is position on asset description in
	// response.Descriptions array
	//
	// We need it for fast asset's description
	// searching in future
	for i, desc := range response.Descriptions {
		key := fmt.Sprintf("%d_%d", desc.ClassID, desc.InstanceID)

		descriptions[key] = i
	}

	for _, asset := range response.Assets {
		key := fmt.Sprintf("%d_%d", asset.ClassID, asset.InstanceID)
		descPos := descriptions[key]

		picts := ItemPict{
			Standard: fmt.Sprintf(PicturePathPattern, response.Descriptions[descPos].IconURL),
		}

		if len(response.Descriptions[descPos].IconURLLarge) != 0 {
			picts.Large = fmt.Sprintf(PicturePathPattern, response.Descriptions[descPos].IconURLLarge)
		}

		item := InventoryItem{
			AppID:       asset.AppID,
			ContextID:   asset.ContextID,
			AssetID:     asset.AssetID,
			ClassID:     asset.ClassID,
			InstanceID:  asset.InstanceID,
			Amount:      asset.Amount,
			Tradable:    response.Descriptions[descPos].Tradable == 1,
			Currency:    response.Descriptions[descPos].Currency,
			Pictures:    picts,
			Description: "",
			Name: ItemName{
				Text:       response.Descriptions[descPos].Name,
				Color:      response.Descriptions[descPos].NameColor,
				MarketText: response.Descriptions[descPos].MarketName,
				MarketHash: response.Descriptions[descPos].MarketHashName,
			},
			Commodity: response.Descriptions[descPos].Commodity == 1,
			Type:      response.Descriptions[descPos].Type,
			Market: ItemMarket{
				Marketable:   response.Descriptions[descPos].Marketable == 1,
				Restrictions: response.Descriptions[descPos].MarketTradableRestriction,
			},
		}

		*items = append(*items, item)
	}

	hasMore = response.HasMore != 0

	if !hasMore {
		return hasMore, 0, nil
	}

	lastAssetID, err = strconv.ParseUint(response.LastAssetID, 10, 64)
	if err != nil {
		return hasMore, 0, err
	}

	return hasMore, lastAssetID, nil
}

func (session *Session) GetInventory(sid SteamID, appID, contextID uint64, lang Lang) ([]InventoryItem, error) {
	items := []InventoryItem{}
	startAssetID := uint64(0)

	for {
		hasMore, lastAssetID, err := session.fetchInventory(sid, appID, contextID, lang, startAssetID, &items)
		if err != nil {
			return nil, err
		}

		if !hasMore {
			break
		}

		startAssetID = lastAssetID
	}

	return items, nil
}

func (session *Session) GetInventoryAppStats(sid SteamID) (map[string]InventoryAppStats, error) {
	resp, err := session.client.Get("https://steamcommunity.com/profiles/" + sid.ToString() + "/inventory")
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	m := inventoryContextRegexp.FindSubmatch(body)
	if m == nil || len(m) != 2 {
		return nil, err
	}

	inven := map[string]InventoryAppStats{}
	if err = json.Unmarshal(m[1], &inven); err != nil {
		return nil, err
	}

	return inven, nil
}
<commit_msg>inventory: use item picture hash only in ItemPict<commit_after>package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/url"
	"regexp"
	"strconv"
)

type Lang string

const (
	LangEng = "english"
	LangRus = "russian"
)

const (
	InventoryEndpoint = "http://steamcommunity.com/inventory/%d/%d/%d"
)

type ItemPict struct {
	Standard string `json:"standard"`
	Large    string `json:"large"`
}

type ItemName struct {
	Text       string `json:"text"`
	Color      string `json:"color"`
	MarketText string `json:"market_text"`
	MarketHash string `json:"market_hash"`
}

type ItemMarket struct {
	Marketable   bool `json:"marketable"`
	Restrictions int  `json:"restrictions"`
}

// Due to the JSON being string, etc... we cannot re-use EconItem
// Also, "assetid" is included as "id" not as assetid.
type InventoryItem struct {
	AppID       uint64     `json:"appid"`
	ContextID   uint64     `json:"contextid"`
	AssetID     uint64     `json:"assetid"`
	ClassID     uint64     `json:"classid"`
	InstanceID  uint64     `json:"instanceid"`
	Amount      uint64     `json:"amount"`
	Tradable    bool       `json:"tradable"`
	Currency    int        `json:"currency"`
	Pictures    ItemPict   `json:"pictures"`
	Description string     `json:"desc"`
	Name        ItemName   `json:"name"`
	Commodity   bool       `json:"commodity"`
	Type        string     `json:"type"`
	Market      ItemMarket `json:"market"`
}

type InventoryContext struct {
	ID         uint64 `json:"id,string"` /* Apparently context id needs at least 64 bits...  */
	AssetCount uint32 `json:"asset_count"`
	Name       string `json:"name"`
}

type InventoryAppStats struct {
	AppID            uint64                       `json:"appid"`
	Name             string                       `json:"name"`
	AssetCount       uint32                       `json:"asset_count"`
	Icon             string                       `json:"icon"`
	Link             string                       `json:"link"`
	InventoryLogo    string                       `json:"inventory_logo"`
	TradePermissions string                       `json:"trade_permissions"`
	Contexts         map[string]*InventoryContext `json:"rgContexts"`
}

var inventoryContextRegexp = regexp.MustCompile("var g_rgAppContextData = (.*?);")

func (session *Session) fetchInventory(sid SteamID, appID, contextID uint64, lang Lang, startAssetID uint64, items *[]InventoryItem) (hasMore bool, lastAssetID uint64, err error) {
	params := url.Values{
		"l":             {string(lang)},
		"start_assetid": {strconv.FormatUint(startAssetID, 10)},
	}

	requestURL := fmt.Sprintf(InventoryEndpoint, sid, appID, contextID)

	resp, err := session.client.Get(requestURL + "?" + params.Encode())
	if err != nil {
		return false, 0, err
	}

	type Asset struct {
		AppID      uint64 `json:"appid,string"`
		ContextID  uint64 `json:"contextid,string"`
		AssetID    uint64 `json:"assetid,string"`
		ClassID    uint64 `json:"classid,string"`
		InstanceID uint64 `json:"instanceid,string"`
		Amount     uint64 `json:"amount,string"`
	}

	type DescriptionsPart struct {
		Type  string `json:"type"`
		Value string `json:"value"`
		Color string `json:"color"`
	}

	type Description struct {
		AppID                     uint64             `json:"appid"`
		ClassID                   uint64             `json:"classid,string"`
		InstanceID                uint64             `json:"instanceid,string"`
		Currency                  int                `json:"currency"`
		BackgroundColor           string             `json:"background_color"`
		IconURL                   string             `json:"icon_url"`
		IconURLLarge              string             `json:"icon_url_large"`
		Descriptions              []DescriptionsPart `json:"descriptions"`
		Tradable                  int                `json:"tradable"`
		Name                      string             `json:"name"`
		NameColor                 string             `json:"name_color"`
		Type                      string             `json:"type"`
		MarketName                string             `json:"market_name"`
		MarketHashName            string             `json:"market_hash_name"`
		Commodity                 int                `json:"commodity"`
		MarketTradableRestriction int                `json:"market_tradable_restriction"`
		Marketable                int                `json:"marketable"`
	}

	type Response struct {
		Assets              []Asset       `json:"assets"`
		Descriptions        []Description `json:"descriptions"`
		Success             int           `json:"success"`
		HasMore             int           `json:"more_items"`
		LastAssetID         string        `json:"last_assetid"`
		TotalInventoryCount int           `json:"total_inventory_count"`
		ErrorMsg            string        `json:"error"`
	}

	var response Response
	if err = json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return false, 0, err
	}

	if response.Success == 0 {
		if len(response.ErrorMsg) != 0 {
			return false, 0, errors.New(response.ErrorMsg)
		}

		return false, 0, nil // empty inventory
	}

	descriptions := make(map[string]int)

	// Fill in descriptions map, where key
	// is "<CLASS_ID>_<INSTANCE_ID>" pattern, and
	// value is position on asset description in
	// response.Descriptions array
	//
	// We need it for fast asset's description
	// searching in future
	for i, desc := range response.Descriptions {
		key := fmt.Sprintf("%d_%d", desc.ClassID, desc.InstanceID)

		descriptions[key] = i
	}

	for _, asset := range response.Assets {
		key := fmt.Sprintf("%d_%d", asset.ClassID, asset.InstanceID)
		descPos := descriptions[key]

		picts := ItemPict{
			Standard: response.Descriptions[descPos].IconURL,
		}

		if len(response.Descriptions[descPos].IconURLLarge) != 0 {
			picts.Large = response.Descriptions[descPos].IconURLLarge
		}

		item := InventoryItem{
			AppID:       asset.AppID,
			ContextID:   asset.ContextID,
			AssetID:     asset.AssetID,
			ClassID:     asset.ClassID,
			InstanceID:  asset.InstanceID,
			Amount:      asset.Amount,
			Tradable:    response.Descriptions[descPos].Tradable == 1,
			Currency:    response.Descriptions[descPos].Currency,
			Pictures:    picts,
			Description: "",
			Name: ItemName{
				Text:       response.Descriptions[descPos].Name,
				Color:      response.Descriptions[descPos].NameColor,
				MarketText: response.Descriptions[descPos].MarketName,
				MarketHash: response.Descriptions[descPos].MarketHashName,
			},
			Commodity: response.Descriptions[descPos].Commodity == 1,
			Type:      response.Descriptions[descPos].Type,
			Market: ItemMarket{
				Marketable:   response.Descriptions[descPos].Marketable == 1,
				Restrictions: response.Descriptions[descPos].MarketTradableRestriction,
			},
		}

		*items = append(*items, item)
	}

	hasMore = response.HasMore != 0

	if !hasMore {
		return hasMore, 0, nil
	}

	lastAssetID, err = strconv.ParseUint(response.LastAssetID, 10, 64)
	if err != nil {
		return hasMore, 0, err
	}

	return hasMore, lastAssetID, nil
}

func (session *Session) GetInventory(sid SteamID, appID, contextID uint64, lang Lang) ([]InventoryItem, error) {
	items := []InventoryItem{}
	startAssetID := uint64(0)

	for {
		hasMore, lastAssetID, err := session.fetchInventory(sid, appID, contextID, lang, startAssetID, &items)
		if err != nil {
			return nil, err
		}

		if !hasMore {
			break
		}

		startAssetID = lastAssetID
	}

	return items, nil
}

func (session *Session) GetInventoryAppStats(sid SteamID) (map[string]InventoryAppStats, error) {
	resp, err := session.client.Get("https://steamcommunity.com/profiles/" + sid.ToString() + "/inventory")
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	m := inventoryContextRegexp.FindSubmatch(body)
	if m == nil || len(m) != 2 {
		return nil, err
	}

	inven := map[string]InventoryAppStats{}
	if err = json.Unmarshal(m[1], &inven); err != nil {
		return nil, err
	}

	return inven, nil
}
<|endoftext|>


<commit_before>package faroo

import (
	"appengine"
	"appengine/urlfetch"

 	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type Related struct {
	Title	string `json:"title"`
	Url	string `json:"url"`
	Domain	string `json:"domain"`
}

type Entry struct {
	Title	string `json:"title"`
	Kwic	string `json:"kwic"`
	Content string `json:"content"`
	Url	string `json:"url"`
	Iurl	string `json:"iurl"`
	Domain	string `json:"domain"`
	Authoor string `json:"author"`
	Date	int64 `json:"date"`
}

type EntryList struct {
	Results []Entry `json:"results"`
	Query	string `json:"query"`
	Suggestions []string `json:"suggestions"`
	Count int `json:"count"`
	Start int `json:"start"`
	Length int `json:"length"`
	Time string `json:"time"`
}

type PushToken struct {
	CreatedAt string `json:"createdAt"`
	CustomizedTopic1 string  `json:"mCustomizedTopic1"`
	CustomizedTopic2 string `json:"mCustomizedTopic2"`
	CustomizedTopic3 string `json:"mCustomizedTopic3"`
	Language string  `json:"mLanguage"`
	DeviceId string `json:"mDeviceId"`
	GoogleId string `json:"mGoogleId"`
	PushToken string `json:"mPushToken"`
	ObjectId string `json:"objectId"`
	UpdatedAt string `json:"updatedAt"`
}

type PushTokenList struct {
	Results []PushToken  `json:"results"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
		http.HandleFunc("/pushEn", pushHandleEnglish)
		http.HandleFunc("/pushDe", pushHandleGerman)
		http.HandleFunc("/pushZh", pushHandleChinese)
		http.HandleFunc("/pushCus", pushHandleCus)
}

func pushHandleEnglish(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_EN)
}

func pushHandleGerman(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_DE)
}

func pushHandleChinese(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_ZH)
}

func pushHandleCus(w http.ResponseWriter, r *http.Request ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	ch := make(chan string)
	go doCus(r, ch)
	pushedResult := <-ch
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

//A push handler on all languages which will be used by calling http://your-app.appspot.com/push
func pushHandle(w http.ResponseWriter, r *http.Request, topicList map[string]Topic ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	topicCount := len(topicList)
	ch := make(chan string, topicCount)
    for k, v := range topicList {
        go doPush(r,k,v, ch)
    }
	var pushedResult string = ""
	for i:=0; i < topicCount; i++ {
		pushedResult += (<-ch)
	}
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

func doPush(r *http.Request, api string, topic Topic, ch chan string) {
	var pushedContent string = ""
	pRes := getEntryList(r, topic.LocalName, topic.Language)
	topicApi := (TOPICS + api)
	if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
		entry := pRes.Results[0]

		entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
		entry.Title = strings.Replace(entry.Title, "%", "％", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

		resBytes, _ := json.Marshal(entry)
		data := string(resBytes)
		pushed := push(r, topicApi, data, true)
		pushedContent += pushed
	} else {
		pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
	}
	pushedContent += ","
	ch <- pushedContent
}

func doCus(r *http.Request, ch chan string) {
	pushTokenList := getPushTokens(r)
	var allPushedContent string = ""
	if pushTokenList.Results != nil && len(pushTokenList.Results) > 0 {
		var sz int = len(pushTokenList.Results)
		out := make(chan string, sz)
		for _, v := range pushTokenList.Results {
    		go func(pushToken PushToken) {
				pRess := [CUSTOMIZED_TOTAL]*EntryList{}
				pRess[0] = getEntryList(r, pushToken.CustomizedTopic1, pushToken.Language)
				pRess[1] = getEntryList(r, pushToken.CustomizedTopic2, pushToken.Language)
				pRess[2] = getEntryList(r, pushToken.CustomizedTopic3, pushToken.Language)

				topicApi := pushToken.PushToken
				var pushedContent string = ""
				for _, pRes := range pRess {
					if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
						entry := pRes.Results[0]

						entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
						entry.Title = strings.Replace(entry.Title, "%", "％", -1)
						entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
						entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

						resBytes, _ := json.Marshal(entry)
						data := string(resBytes)
						pushed := push(r, topicApi, data, true)
						pushedContent += pushed
					} else {
						pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
					}
					pushedContent += ","
				}
				out <- pushedContent
    		}(v)
		}
		for i := 0; i < sz; i++ {
			allPushedContent += <-out
		}
	}
	ch <- allPushedContent
}

//Get all push-tokens of clients inc. the customized topics user subscribed.
func getPushTokens(r *http.Request)(pRes *PushTokenList) {
	cxt := appengine.NewContext(r)
	url := DB_PATH + DB_PUSH_TOKEN_TAB
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		req.Header.Add(DB_HEADER_APP_ID, DB_APP_ID)
		req.Header.Add(DB_HEADER_API_KEY, DB_API_KEY)
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(PushTokenList)
				json.Unmarshal(bytes, pRes)
				cxt.Infof("json: %s", string(bytes))
			} else {
				cxt.Errorf("getPushTokens unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getPushTokens doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getPushTokens: %v", err)
		pRes = nil
	}
	return
}

//Get a news-entry and channel it.
func getEntryList(r *http.Request, query string, lang string)(pRes *EntryList) {
	cxt := appengine.NewContext(r)
	if strings.Contains(query, "global") {
		query = ""
	}
	url := fmt.Sprintf(FAROO_API, query, lang)
	if req, err := http.NewRequest("GET", url + "&rlength=0", nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(EntryList)
				json.Unmarshal(bytes, pRes)
			} else {
				cxt.Errorf("getEntryList unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getEntryList doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getEntryList: %v", err)
		pRes = nil
	}
	return
}

//Make push
func push(r *http.Request, topicApi string, data string, scheduledTask bool) (push string) {
	body := fmt.Sprintf(`{"to" : "%s","data" : %s}`, topicApi, data)
	bodyBytes := bytes.NewBufferString(body)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, bodyBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
			push =  ""
		} else {
			cxt.Infof(body)
			push = body
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
		push = ""
	}
	return
}

 <commit_msg>Let 5 customized topics.<commit_after>package faroo

import (
	"appengine"
	"appengine/urlfetch"

 	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

type Related struct {
	Title	string `json:"title"`
	Url	string `json:"url"`
	Domain	string `json:"domain"`
}

type Entry struct {
	Title	string `json:"title"`
	Kwic	string `json:"kwic"`
	Content string `json:"content"`
	Url	string `json:"url"`
	Iurl	string `json:"iurl"`
	Domain	string `json:"domain"`
	Authoor string `json:"author"`
	Date	int64 `json:"date"`
}

type EntryList struct {
	Results []Entry `json:"results"`
	Query	string `json:"query"`
	Suggestions []string `json:"suggestions"`
	Count int `json:"count"`
	Start int `json:"start"`
	Length int `json:"length"`
	Time string `json:"time"`
}

type PushToken struct {
	CreatedAt string `json:"createdAt"`
	CustomizedTopic1 string `json:"mCustomizedTopic1"`
	CustomizedTopic2 string `json:"mCustomizedTopic2"`
	CustomizedTopic3 string `json:"mCustomizedTopic3"`
	CustomizedTopic4 string `json:"mCustomizedTopic4"`
	CustomizedTopic5 string `json:"mCustomizedTopic5"`
	Language string  `json:"mLanguage"`
	DeviceId string `json:"mDeviceId"`
	GoogleId string `json:"mGoogleId"`
	PushToken string `json:"mPushToken"`
	ObjectId string `json:"objectId"`
	UpdatedAt string `json:"updatedAt"`
}

type PushTokenList struct {
	Results []PushToken  `json:"results"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
		http.HandleFunc("/pushEn", pushHandleEnglish)
		http.HandleFunc("/pushDe", pushHandleGerman)
		http.HandleFunc("/pushZh", pushHandleChinese)
		http.HandleFunc("/pushCus", pushHandleCus)
}

func pushHandleEnglish(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_EN)
}

func pushHandleGerman(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_DE)
}

func pushHandleChinese(w http.ResponseWriter, r *http.Request ) {
	pushHandle(w, r, TOPIC_LIST_ZH)
}

func pushHandleCus(w http.ResponseWriter, r *http.Request ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	ch := make(chan string)
	go doCus(r, ch)
	pushedResult := <-ch
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

//A push handler on all languages which will be used by calling http://your-app.appspot.com/push
func pushHandle(w http.ResponseWriter, r *http.Request, topicList map[string]Topic ) {
	defer func() {
		if err := recover(); err != nil {
			s := fmt.Sprintf(`{"status":%d }`, 300)
			w.Header().Set("Content-Type", API_RESTYPE)
			fmt.Fprintf(w, s)
		}
	}()

	topicCount := len(topicList)
	ch := make(chan string, topicCount)
    for k, v := range topicList {
        go doPush(r,k,v, ch)
    }
	var pushedResult string = ""
	for i:=0; i < topicCount; i++ {
		pushedResult += (<-ch)
	}
	if pushedResult != "" {
		pushedResult = pushedResult[:len(pushedResult)-1]
	}
	output := fmt.Sprintf(`{"status":%d, "pushed:":[%s] }`, 200, pushedResult)
	w.Header().Set("Content-Type", API_RESTYPE)
	fmt.Fprintf(w, output)
}

//Push topics.
func doPush(r *http.Request, api string, topic Topic, ch chan string) {
	var pushedContent string = ""
	pRes := getEntryList(r, topic.LocalName, topic.Language)
	topicApi := (TOPICS + api)
	if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
		entry := pRes.Results[0]

		entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
		entry.Title = strings.Replace(entry.Title, "%", "％", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
		entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

		resBytes, _ := json.Marshal(entry)
		data := string(resBytes)
		pushed := push(r, topicApi, data, true)
		pushedContent += pushed
	} else {
		pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
	}
	pushedContent += ","
	ch <- pushedContent
}

//Push customized things.
func doCus(r *http.Request, ch chan string) {
	pushTokenList := getPushTokens(r)
	if pushTokenList == nil {
		ch <- `"Error(nil) in getting push-token""`
		return
	}
	var allPushedContent string = ""
	if pushTokenList.Results != nil && len(pushTokenList.Results) > 0 {
		var sz int = len(pushTokenList.Results)
		out := make(chan string, sz)
		for _, v := range pushTokenList.Results {
    		go func(pushToken PushToken) {
				pRess := [CUSTOMIZED_TOTAL]*EntryList{}
				if pushToken.CustomizedTopic1 != "" {
					pRess[0] = getEntryList(r, pushToken.CustomizedTopic1, pushToken.Language)
				} else {
					pRess[0] = nil
				}
				if pushToken.CustomizedTopic2 != "" {
					pRess[1] = getEntryList(r, pushToken.CustomizedTopic2, pushToken.Language)
				} else {
					pRess[1] = nil
				}
				if pushToken.CustomizedTopic3 != "" {
					pRess[2] = getEntryList(r, pushToken.CustomizedTopic3, pushToken.Language)
				} else {
					pRess[2] = nil
				}
				if pushToken.CustomizedTopic4 != "" {
					pRess[3] = getEntryList(r, pushToken.CustomizedTopic4, pushToken.Language)
				} else {
					pRess[3] = nil
				}
				if pushToken.CustomizedTopic5 != "" {
					pRess[4] = getEntryList(r, pushToken.CustomizedTopic5, pushToken.Language)
				} else {
					pRess[4] = nil
				}
				topicApi := pushToken.PushToken
				var pushedContent string = ""
				for _, pRes := range pRess {
					if pRes != nil && pRes.Results != nil &&  len(pRes.Results) > 0 {
						entry := pRes.Results[0]

						entry.Title = strings.Replace(entry.Title, "\"", "'", -1)
						entry.Title = strings.Replace(entry.Title, "%", "％", -1)
						entry.Kwic = strings.Replace(entry.Kwic, "\"", "'", -1)
						entry.Kwic = strings.Replace(entry.Kwic, "%", "％", -1)

						resBytes, _ := json.Marshal(entry)
						data := string(resBytes)
						pushed := push(r, topicApi, data, true)
						pushedContent += pushed
					} else {
						pushedContent += (fmt.Sprintf(`"failed in %s"`, topicApi))
					}
					pushedContent += ","
				}
				out <- pushedContent
    		}(v)
		}
		for i := 0; i < sz; i++ {
			allPushedContent += <-out
		}
	}
	ch <- allPushedContent
}

//Get all push-tokens of clients inc. the customized topics user subscribed.
func getPushTokens(r *http.Request)(pRes *PushTokenList) {
	cxt := appengine.NewContext(r)
	url := DB_PATH + DB_PUSH_TOKEN_TAB
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		req.Header.Add(DB_HEADER_APP_ID, DB_APP_ID)
		req.Header.Add(DB_HEADER_API_KEY, DB_API_KEY)
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(PushTokenList)
				json.Unmarshal(bytes, pRes)
				cxt.Infof("json: %s", string(bytes))
			} else {
				cxt.Errorf("getPushTokens unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getPushTokens doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getPushTokens: %v", err)
		pRes = nil
	}
	return
}

//Get a news-entry and channel it.
func getEntryList(r *http.Request, query string, lang string)(pRes *EntryList) {
	cxt := appengine.NewContext(r)
	if strings.Contains(query, "global") {
		query = ""
	}
	url := fmt.Sprintf(FAROO_API, query, lang)
	if req, err := http.NewRequest("GET", url + "&rlength=0", nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pRes = new(EntryList)
				json.Unmarshal(bytes, pRes)
			} else {
				cxt.Errorf("getEntryList unmarshal: %v", err)
				pRes = nil
			}
		} else {
			cxt.Errorf("getEntryList doing: %v", err)
			pRes = nil
		}
	} else {
		cxt.Errorf("getEntryList: %v", err)
		pRes = nil
	}
	return
}

//Make push
func push(r *http.Request, topicApi string, data string, scheduledTask bool) (push string) {
	body := fmt.Sprintf(`{"to" : "%s","data" : %s}`, topicApi, data)
	bodyBytes := bytes.NewBufferString(body)
	cxt := appengine.NewContext(r)
	if req, err := http.NewRequest("POST", PUSH_SENDER, bodyBytes); err == nil {
		req.Header.Add("Authorization", PUSH_KEY)
		req.Header.Add("Content-Type", API_RESTYPE)
		if scheduledTask {
			req.Header.Add("X-AppEngine-Cron", "true")
		}
		client := urlfetch.Client(cxt)
		res, _ := client.Do(req)
		if res != nil {
			defer res.Body.Close()
		}
		if err != nil {
			cxt.Errorf("Push summary doing: %v", err)
			push =  ""
		} else {
			cxt.Infof(body)
			push = body
		}
	} else {
		cxt.Errorf("Push summary: %v", err)
		push = ""
	}
	return
}

 <|endoftext|>


<commit_before>// © 2012 the Minima Authors under the MIT license. See AUTHORS for the list of authors.

package item

import (
	"code.google.com/p/min-game/geom"
	"code.google.com/p/min-game/world"
	"fmt"
)

const (
	ETele   = "E-Tele"
	Uranium = "Uranium"
)

// InitialUses returns the initial value for the Uses field
// of items, indexed by their name.
var initialUses = map[string]int{
	ETele: 3,
}

// Descs maps item names to functions that return
// an item description.
var descs = map[string]func(*Item) string{
	ETele: withUses(`The Emergency Teleporter reacts to
		critical condition by sending you back to home base`),

	Uranium: func(*Item) string {
		return `Uranium is of great interest because of its application to
		nuclear power and nuclear weapons. Uranium contamination is
		an emotive environmental problem. It is not particularly rare and
		is more common than beryllium or tungsten for instance.
		[br] [br] http://www.webelements.com/uranium`
	},
}

// WithUses returns a function that prints the description
// followed by the remaining number of uses.
func withUses(desc string) func(*Item) string {
	return func(i *Item) string {
		return fmt.Sprintf("%s  This %s currently has %d uses remaining.",
			desc, i.Name, i.Uses)
	}
}

// An Item is something that the player can collect and possibly use.
type Item struct {
	Name string
	Uses int
}

// New returns a new item of the given name.
func New(name string) *Item {
	return &Item{name, initialUses[name]}
}

// Desc returns the item's description.
func (it *Item) Desc() string {
	if d, ok := descs[it.Name]; ok {
		return d(it)
	}
	return "<No Description for " + it.Name + ">"
}

// A Treasure is an Item located somewhere in the world.
type Treasure struct {
	Item *Item
	Box  geom.Rectangle
}

// NewTreasure returns a new treasure.
func NewTreasure(x, y float64, it *Item) *Treasure {
	return &Treasure{
		Item: it,
		Box:  geom.Rect(x, y, x+world.TileSize.X, y+world.TileSize.Y),
	}
}
<commit_msg>Add missing period to e-tele description.<commit_after>// © 2012 the Minima Authors under the MIT license. See AUTHORS for the list of authors.

package item

import (
	"code.google.com/p/min-game/geom"
	"code.google.com/p/min-game/world"
	"fmt"
)

const (
	ETele   = "E-Tele"
	Uranium = "Uranium"
)

// InitialUses returns the initial value for the Uses field
// of items, indexed by their name.
var initialUses = map[string]int{
	ETele: 3,
}

// Descs maps item names to functions that return
// an item description.
var descs = map[string]func(*Item) string{
	ETele: withUses(`The Emergency Teleporter reacts to
		critical condition by sending you back to home base.`),

	Uranium: func(*Item) string {
		return `Uranium is of great interest because of its application to
		nuclear power and nuclear weapons. Uranium contamination is
		an emotive environmental problem. It is not particularly rare and
		is more common than beryllium or tungsten for instance.
		[br] [br] http://www.webelements.com/uranium`
	},
}

// WithUses returns a function that prints the description
// followed by the remaining number of uses.
func withUses(desc string) func(*Item) string {
	return func(i *Item) string {
		return fmt.Sprintf("%s  This %s currently has %d uses remaining.",
			desc, i.Name, i.Uses)
	}
}

// An Item is something that the player can collect and possibly use.
type Item struct {
	Name string
	Uses int
}

// New returns a new item of the given name.
func New(name string) *Item {
	return &Item{name, initialUses[name]}
}

// Desc returns the item's description.
func (it *Item) Desc() string {
	if d, ok := descs[it.Name]; ok {
		return d(it)
	}
	return "<No Description for " + it.Name + ">"
}

// A Treasure is an Item located somewhere in the world.
type Treasure struct {
	Item *Item
	Box  geom.Rectangle
}

// NewTreasure returns a new treasure.
func NewTreasure(x, y float64, it *Item) *Treasure {
	return &Treasure{
		Item: it,
		Box:  geom.Rect(x, y, x+world.TileSize.X, y+world.TileSize.Y),
	}
}
<|endoftext|>


<commit_before>package kite

import (
	"errors"
	"fmt"
	"koding/newkite/dnode"
	"koding/newkite/protocol"
	"net"
	"sync"
)

// Kontrol embeds RemoteKite which has additional special helper methods.
type Kontrol struct {
	RemoteKite

	// used for synchronizing methods that needs to be called after
	// successful connection.
	ready chan bool
}

// NewKontrol returns a pointer to new Kontrol instance.
func (k *Kite) NewKontrol(addr string) *Kontrol {
	// Only the address is required to connect Kontrol
	host, port, _ := net.SplitHostPort(addr)
	kite := protocol.Kite{
		PublicIP: host,
		Port:     port,
		Name:     "kontrol", // for logging purposes
	}

	auth := callAuthentication{
		Type: "kodingKey",
		Key:  k.KodingKey,
	}

	remoteKite := k.NewRemoteKite(kite, auth)
	remoteKite.client.Reconnect = true

	var once sync.Once
	ready := make(chan bool)

	remoteKite.OnConnect(func() {
		k.Log.Info("Connected to Kontrol ")

		// signal all other methods that are listening on this channel, that we
		// are ready.
		once.Do(func() { close(ready) })
	})

	remoteKite.OnDisconnect(func() { k.Log.Warning("Disconnected from Kontrol. I will retry in background...") })

	return &Kontrol{
		RemoteKite: *remoteKite,
		ready:      ready,
	}
}

// Register registers current Kite to Kontrol. After registration other Kites
// can find it via GetKites() method.
func (k *Kontrol) Register() error {
	response, err := k.RemoteKite.Call("register", nil)
	if err != nil {
		return err
	}

	var rr protocol.RegisterResult
	err = response.Unmarshal(&rr)
	if err != nil {
		return err
	}

	switch rr.Result {
	case protocol.AllowKite:
		kite := &k.localKite.Kite

		// we know now which user that is after authentication
		kite.Username = rr.Username

		// Set the correct PublicIP if left empty in options.
		if kite.PublicIP == "" {
			kite.PublicIP = rr.PublicIP
		}

		k.Log.Info("Registered to kontrol with addr: %s version: %s uuid: %s",
			kite.Addr(), kite.Version, kite.ID)
	case protocol.RejectKite:
		return errors.New("Kite rejected")
	default:
		return fmt.Errorf("Invalid result: %s", rr.Result)
	}

	return nil
}

// GetKites returns the list of Kites matching the query.
// The returned list contains ready to connect RemoteKite instances.
// The caller must connect with RemoteKite.Dial() before using each Kite.
func (k *Kontrol) GetKites(query protocol.KontrolQuery, onEvent func(*protocol.KiteEvent)) ([]*RemoteKite, error) {
	// this is needed because we are calling GetKites explicitly, therefore
	// this should be only callable *after* we are connected to kontrol.
	<-k.ready

	queueEvents := func(p *dnode.Partial) {
		var args []*dnode.Partial
		err := p.Unmarshal(&args)
		if err != nil {
			k.Log.Error(err.Error())
			return
		}

		if len(args) != 1 {
			k.Log.Error("Invalid Kite event")
			return
		}

		var event protocol.KiteEvent
		err = args[0].Unmarshal(&event)
		if err != nil {
			k.Log.Error(err.Error())
			return
		}

		onEvent(&event)
	}

	args := []interface{}{query}
	if onEvent != nil {
		args = append(args, dnode.Callback(queueEvents))
	}

	response, err := k.RemoteKite.Call("getKites", args)
	if err != nil {
		return nil, err
	}

	var kites []protocol.KiteWithToken
	err = response.Unmarshal(&kites)
	if err != nil {
		return nil, err
	}

	remoteKites := make([]*RemoteKite, len(kites))
	for i, kite := range kites {
		auth := callAuthentication{
			Type: "token",
			Key:  kite.Token,
		}

		remoteKites[i] = k.localKite.NewRemoteKite(kite.Kite, auth)
	}

	return remoteKites, nil
}

func (k *Kontrol) GetToken(kite *protocol.Kite) (string, error) {
	<-k.ready

	result, err := k.RemoteKite.Call("getToken", kite)
	if err != nil {
		return "", err
	}

	return result.MustString(), nil
}
<commit_msg>kite: fix reconnect to kontrol bug<commit_after>package kite

import (
	"errors"
	"fmt"
	"koding/newkite/dnode"
	"koding/newkite/protocol"
	"net"
	"sync"
)

// Kontrol embeds RemoteKite which has additional special helper methods.
type Kontrol struct {
	*RemoteKite

	// used for synchronizing methods that needs to be called after
	// successful connection.
	ready chan bool
}

// NewKontrol returns a pointer to new Kontrol instance.
func (k *Kite) NewKontrol(addr string) *Kontrol {
	// Only the address is required to connect Kontrol
	host, port, _ := net.SplitHostPort(addr)
	kite := protocol.Kite{
		PublicIP: host,
		Port:     port,
		Name:     "kontrol", // for logging purposes
	}

	auth := callAuthentication{
		Type: "kodingKey",
		Key:  k.KodingKey,
	}

	remoteKite := k.NewRemoteKite(kite, auth)
	remoteKite.client.Reconnect = true

	var once sync.Once
	ready := make(chan bool)

	remoteKite.OnConnect(func() {
		k.Log.Info("Connected to Kontrol ")

		// signal all other methods that are listening on this channel, that we
		// are ready.
		once.Do(func() { close(ready) })
	})

	remoteKite.OnDisconnect(func() { k.Log.Warning("Disconnected from Kontrol. I will retry in background...") })

	return &Kontrol{
		RemoteKite: remoteKite,
		ready:      ready,
	}
}

// Register registers current Kite to Kontrol. After registration other Kites
// can find it via GetKites() method.
func (k *Kontrol) Register() error {
	response, err := k.RemoteKite.Call("register", nil)
	if err != nil {
		return err
	}

	var rr protocol.RegisterResult
	err = response.Unmarshal(&rr)
	if err != nil {
		return err
	}

	switch rr.Result {
	case protocol.AllowKite:
		kite := &k.localKite.Kite

		// we know now which user that is after authentication
		kite.Username = rr.Username

		// Set the correct PublicIP if left empty in options.
		if kite.PublicIP == "" {
			kite.PublicIP = rr.PublicIP
		}

		k.Log.Info("Registered to kontrol with addr: %s version: %s uuid: %s",
			kite.Addr(), kite.Version, kite.ID)
	case protocol.RejectKite:
		return errors.New("Kite rejected")
	default:
		return fmt.Errorf("Invalid result: %s", rr.Result)
	}

	return nil
}

// GetKites returns the list of Kites matching the query.
// The returned list contains ready to connect RemoteKite instances.
// The caller must connect with RemoteKite.Dial() before using each Kite.
func (k *Kontrol) GetKites(query protocol.KontrolQuery, onEvent func(*protocol.KiteEvent)) ([]*RemoteKite, error) {
	// this is needed because we are calling GetKites explicitly, therefore
	// this should be only callable *after* we are connected to kontrol.
	<-k.ready

	queueEvents := func(p *dnode.Partial) {
		var args []*dnode.Partial
		err := p.Unmarshal(&args)
		if err != nil {
			k.Log.Error(err.Error())
			return
		}

		if len(args) != 1 {
			k.Log.Error("Invalid Kite event")
			return
		}

		var event protocol.KiteEvent
		err = args[0].Unmarshal(&event)
		if err != nil {
			k.Log.Error(err.Error())
			return
		}

		onEvent(&event)
	}

	args := []interface{}{query}
	if onEvent != nil {
		args = append(args, dnode.Callback(queueEvents))
	}

	response, err := k.RemoteKite.Call("getKites", args)
	if err != nil {
		return nil, err
	}

	var kites []protocol.KiteWithToken
	err = response.Unmarshal(&kites)
	if err != nil {
		return nil, err
	}

	remoteKites := make([]*RemoteKite, len(kites))
	for i, kite := range kites {
		auth := callAuthentication{
			Type: "token",
			Key:  kite.Token,
		}

		remoteKites[i] = k.localKite.NewRemoteKite(kite.Kite, auth)
	}

	return remoteKites, nil
}

func (k *Kontrol) GetToken(kite *protocol.Kite) (string, error) {
	<-k.ready

	result, err := k.RemoteKite.Call("getToken", kite)
	if err != nil {
		return "", err
	}

	return result.MustString(), nil
}
<|endoftext|>