<commit_before>package java

import (
	"testing"

	"github.com/anchore/syft/syft/pkg"
	"github.com/sergi/go-diff/diffmatchpatch"
)

func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
	tests := []struct {
		filename  string
		version   string
		extension string
		name      string
		ty        pkg.Type
	}{
		{
			filename:  "pkg-maven-4.3.2.blerg",
			version:   "4.3.2",
			extension: "blerg",
			name:      "pkg-maven",
			ty:        pkg.UnknownPkg,
		},
		{
			filename:  "pkg-maven-4.3.2.jar",
			version:   "4.3.2",
			extension: "jar",
			name:      "pkg-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "pkg-extra-field-maven-4.3.2.war",
			version:   "4.3.2",
			extension: "war",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "pkg-extra-field-maven-4.3.2-rc1.ear",
			version:   "4.3.2-rc1",
			extension: "ear",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path/pkg-extra-field-maven-4.3.2-rc1.jpi",
			version:   "4.3.2-rc1",
			extension: "jpi",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JenkinsPluginPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/pkg-extra-field-maven-4.3.2-rc1.hpi",
			version:   "4.3.2-rc1",
			extension: "hpi",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JenkinsPluginPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3a-hudson.jar",
			version:   "1.0.2-beta-2.2.3a-hudson",
			extension: "jar",
			name:      "wagon-webdav",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3-hudson.jar",
			version:   "1.0.2-beta-2.2.3-hudson",
			extension: "jar",
			name:      "wagon-webdav",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/windows-remote-command-1.0.jar",
			version:   "1.0",
			extension: "jar",
			name:      "windows-remote-command",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-http-lightweight-1.0.5-beta-2.jar",
			version:   "1.0.5-beta-2",
			extension: "jar",
			name:      "wagon-http-lightweight",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/commons-jelly-1.1-hudson-20100305.jar",
			version:   "1.1-hudson-20100305",
			extension: "jar",
			name:      "commons-jelly",
			ty:        pkg.JavaPkg,
		},
		{
			filename: "/hudson.war:WEB-INF/lib/jtidy-4aug2000r7-dev-hudson-1.jar",
			// I don't see how we can reliably account for this case
			//version:   "4aug2000r7-dev-hudson-1",
			version:   "",
			extension: "jar",
			name:      "jtidy",
			ty:        pkg.JavaPkg,
		},
		{
			filename: "/hudson.war:WEB-INF/lib/trilead-ssh2-build212-hudson-5.jar",
			// I don't see how we can reliably account for this case
			//version:   "build212-hudson-5",
			version:   "5",
			extension: "jar",
			// name: "trilead-ssh2",
			name: "trilead-ssh2-build212-hudson",
			ty:   pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/guava-r06.jar",
			version:   "r06",
			extension: "jar",
			name:      "guava",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "BOOT-INF/lib/spring-data-r2dbc-1.1.0.RELEASE.jar", // Regression: https://github.com/anchore/syft/issues/255
			version:   "1.1.0.RELEASE",
			extension: "jar",
			name:      "spring-data-r2dbc",
			ty:        pkg.JavaPkg,
		},
	}

	for _, test := range tests {
		t.Run(test.filename, func(t *testing.T) {
			obj := newJavaArchiveFilename(test.filename)

			ty := obj.pkgType()
			if ty != test.ty {
				t.Errorf("mismatched type: %+v != %v", ty, test.ty)
			}

			version := obj.version()
			if version != test.version {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(version, test.version, true)
				t.Errorf("mismatched version:\n%s", dmp.DiffPrettyText(diffs))
			}

			extension := obj.extension()
			if extension != test.extension {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(extension, test.extension, true)
				t.Errorf("mismatched extension:\n%s", dmp.DiffPrettyText(diffs))
			}

			name := obj.name()
			if name != test.name {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(name, test.name, true)
				t.Errorf("mismatched name:\n%s", dmp.DiffPrettyText(diffs))
			}
		})
	}
}
<commit_msg>Update existing archive test cases to correct names and versions<commit_after>package java

import (
	"testing"

	"github.com/anchore/syft/syft/pkg"
	"github.com/sergi/go-diff/diffmatchpatch"
)

func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
	tests := []struct {
		filename  string
		version   string
		extension string
		name      string
		ty        pkg.Type
	}{
		{
			filename:  "pkg-maven-4.3.2.blerg",
			version:   "4.3.2",
			extension: "blerg",
			name:      "pkg-maven",
			ty:        pkg.UnknownPkg,
		},
		{
			filename:  "pkg-maven-4.3.2.jar",
			version:   "4.3.2",
			extension: "jar",
			name:      "pkg-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "pkg-extra-field-maven-4.3.2.war",
			version:   "4.3.2",
			extension: "war",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "pkg-extra-field-maven-4.3.2-rc1.ear",
			version:   "4.3.2-rc1",
			extension: "ear",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path/pkg-extra-field-maven-4.3.2-rc1.jpi",
			version:   "4.3.2-rc1",
			extension: "jpi",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JenkinsPluginPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/pkg-extra-field-maven-4.3.2-rc1.hpi",
			version:   "4.3.2-rc1",
			extension: "hpi",
			name:      "pkg-extra-field-maven",
			ty:        pkg.JenkinsPluginPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3a-hudson.jar",
			version:   "1.0.2-beta-2.2.3a-hudson",
			extension: "jar",
			name:      "wagon-webdav",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3-hudson.jar",
			version:   "1.0.2-beta-2.2.3-hudson",
			extension: "jar",
			name:      "wagon-webdav",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/windows-remote-command-1.0.jar",
			version:   "1.0",
			extension: "jar",
			name:      "windows-remote-command",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/some/path-with-version-5.4.3/wagon-http-lightweight-1.0.5-beta-2.jar",
			version:   "1.0.5-beta-2",
			extension: "jar",
			name:      "wagon-http-lightweight",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/commons-jelly-1.1-hudson-20100305.jar",
			version:   "1.1-hudson-20100305",
			extension: "jar",
			name:      "commons-jelly",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/jtidy-4aug2000r7-dev-hudson-1.jar",
			version:   "4aug2000r7-dev-hudson-1",
			extension: "jar",
			name:      "jtidy",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/trilead-ssh2-build212-hudson-5.jar",
			version:   "build212-hudson-5",
			extension: "jar",
			name:      "trilead-ssh2",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "/hudson.war:WEB-INF/lib/guava-r06.jar",
			version:   "r06",
			extension: "jar",
			name:      "guava",
			ty:        pkg.JavaPkg,
		},
		{
			filename:  "BOOT-INF/lib/spring-data-r2dbc-1.1.0.RELEASE.jar", // Regression: https://github.com/anchore/syft/issues/255
			version:   "1.1.0.RELEASE",
			extension: "jar",
			name:      "spring-data-r2dbc",
			ty:        pkg.JavaPkg,
		},
	}

	for _, test := range tests {
		t.Run(test.filename, func(t *testing.T) {
			obj := newJavaArchiveFilename(test.filename)

			ty := obj.pkgType()
			if ty != test.ty {
				t.Errorf("mismatched type: %+v != %v", ty, test.ty)
			}

			version := obj.version()
			if version != test.version {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(version, test.version, true)
				t.Errorf("mismatched version:\n%s", dmp.DiffPrettyText(diffs))
			}

			extension := obj.extension()
			if extension != test.extension {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(extension, test.extension, true)
				t.Errorf("mismatched extension:\n%s", dmp.DiffPrettyText(diffs))
			}

			name := obj.name()
			if name != test.name {
				dmp := diffmatchpatch.New()
				diffs := dmp.DiffMain(name, test.name, true)
				t.Errorf("mismatched name:\n%s", dmp.DiffPrettyText(diffs))
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"io"
	"log"
	"os"
	"time"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

var state = NewState()
var Docker *docker.Client

func main() {
	go attachServer()
	go rpcServer()

	id := randomID()
	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne-rpc."+id, "1113", nil); err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne-attach."+id, "1114", nil); err != nil {
		log.Fatal(err)
	}

	services := disc.Services("flynn-sampi")
	time.Sleep(100 * time.Millisecond) // HAX: remove this when Online is blocking
	schedulers := services.Online()
	if len(schedulers) == 0 {
		log.Fatal("No sampi instances found")
	}

	scheduler, err := rpcplus.DialHTTP("tcp", schedulers[0].Addr)
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Connected to scheduler")

	Docker, err = docker.NewClient("http://localhost:4243")
	if err != nil {
		log.Fatal(err)
	}

	go streamEvents(Docker)
	go syncScheduler(scheduler)

	host := sampi.Host{
		ID:        id,
		Resources: map[string]sampi.ResourceValue{"memory": sampi.ResourceValue{Value: 1024}},
	}

	jobs := make(chan *sampi.Job)
	scheduler.StreamGo("Scheduler.RegisterHost", host, jobs)
	log.Print("Host registered")
	for job := range jobs {
		log.Printf("%#v", job.Config)
		state.AddJob(job)
		container, err := Docker.CreateContainer(job.Config)
		if err == docker.ErrNoSuchImage {
			err = Docker.PullImage(docker.PullImageOptions{Repository: job.Config.Image}, os.Stdout)
			if err != nil {
				log.Fatal(err)
			}
			container, err = Docker.CreateContainer(job.Config)
		}
		if err != nil {
			log.Fatal(err)
		}
		state.SetContainerID(job.ID, container.ID)
		state.WaitAttach(job.ID)
		if err := Docker.StartContainer(container.ID, nil); err != nil {
			log.Fatal(err)
		}
		state.SetStatusRunning(job.ID)
	}
}

func syncScheduler(client *rpcplus.Client) {
	events := make(chan lorne.Event)
	state.AddListener("all", events)
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		log.Println("remove job", event.JobID)
		if err := client.Call("Scheduler.RemoveJobs", []string{event.JobID}, &struct{}{}); err != nil {
			log.Println("remove job", event.JobID, "error:", err)
			// TODO: try to reconnect?
		}
	}
}

func streamEvents(client *docker.Client) {
	stream, err := client.Events()
	if err != nil {
		log.Fatal(err)
	}
	for event := range stream.Events {
		log.Printf("%#v", event)
		if event.Status != "die" {
			continue
		}
		container, err := client.InspectContainer(event.ID)
		if err != nil {
			log.Println("inspect container", event.ID, "error:", err)
			// TODO: set job status anyway?
			continue
		}
		state.SetStatusDone(event.ID, container.State.ExitCode)
	}
	log.Println("events done", stream.Error)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}
<commit_msg>Very broken port allocator<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"io"
	"log"
	"os"
	"strconv"
	"time"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/lorne/types"
	"github.com/flynn/rpcplus"
	"github.com/flynn/sampi/types"
	"github.com/titanous/go-dockerclient"
)

var state = NewState()
var Docker *docker.Client

func main() {
	go attachServer()
	go rpcServer()
	go allocatePorts()

	id := randomID()
	disc, err := discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne-rpc."+id, "1113", nil); err != nil {
		log.Fatal(err)
	}
	if err := disc.Register("flynn-lorne-attach."+id, "1114", nil); err != nil {
		log.Fatal(err)
	}

	services := disc.Services("flynn-sampi")
	time.Sleep(100 * time.Millisecond) // HAX: remove this when Online is blocking
	schedulers := services.Online()
	if len(schedulers) == 0 {
		log.Fatal("No sampi instances found")
	}

	scheduler, err := rpcplus.DialHTTP("tcp", schedulers[0].Addr)
	if err != nil {
		log.Fatal(err)
	}
	log.Print("Connected to scheduler")

	Docker, err = docker.NewClient("http://localhost:4243")
	if err != nil {
		log.Fatal(err)
	}

	go streamEvents(Docker)
	go syncScheduler(scheduler)

	host := sampi.Host{
		ID:        id,
		Resources: map[string]sampi.ResourceValue{"memory": sampi.ResourceValue{Value: 1024}},
	}

	jobs := make(chan *sampi.Job)
	scheduler.StreamGo("Scheduler.RegisterHost", host, jobs)
	log.Print("Host registered")
	for job := range jobs {
		log.Printf("%#v", job.Config)
		if job.TCPPorts > 0 {
			port := strconv.Itoa(<-portAllocator)
			job.Config.Env = append(job.Config.Env, "PORT="+port)
			job.Config.PortSpecs = []string{port + ":" + port}
		}
		state.AddJob(job)
		container, err := Docker.CreateContainer(job.Config)
		if err == docker.ErrNoSuchImage {
			err = Docker.PullImage(docker.PullImageOptions{Repository: job.Config.Image}, os.Stdout)
			if err != nil {
				log.Fatal(err)
			}
			container, err = Docker.CreateContainer(job.Config)
		}
		if err != nil {
			log.Fatal(err)
		}
		state.SetContainerID(job.ID, container.ID)
		state.WaitAttach(job.ID)
		if err := Docker.StartContainer(container.ID, nil); err != nil {
			log.Fatal(err)
		}
		state.SetStatusRunning(job.ID)
	}
}

func syncScheduler(client *rpcplus.Client) {
	events := make(chan lorne.Event)
	state.AddListener("all", events)
	for event := range events {
		if event.Event != "stop" {
			continue
		}
		log.Println("remove job", event.JobID)
		if err := client.Call("Scheduler.RemoveJobs", []string{event.JobID}, &struct{}{}); err != nil {
			log.Println("remove job", event.JobID, "error:", err)
			// TODO: try to reconnect?
		}
	}
}

func streamEvents(client *docker.Client) {
	stream, err := client.Events()
	if err != nil {
		log.Fatal(err)
	}
	for event := range stream.Events {
		log.Printf("%#v", event)
		if event.Status != "die" {
			continue
		}
		container, err := client.InspectContainer(event.ID)
		if err != nil {
			log.Println("inspect container", event.ID, "error:", err)
			// TODO: set job status anyway?
			continue
		}
		state.SetStatusDone(event.ID, container.State.ExitCode)
	}
	log.Println("events done", stream.Error)
}

func randomID() string {
	b := make([]byte, 16)
	enc := make([]byte, 24)
	_, err := io.ReadFull(rand.Reader, b)
	if err != nil {
		panic(err) // This shouldn't ever happen, right?
	}
	base64.URLEncoding.Encode(enc, b)
	return string(bytes.TrimRight(enc, "="))
}

var portAllocator = make(chan int)

// TODO: fix this, horribly broken
const startPort = 55000
const endPort = 65535

func allocatePorts() {
	for i := startPort; i < endPort; i++ {
		portAllocator <- i
	}
	// TODO: handle wrap-around
}
<|endoftext|>


<commit_before>package gdrj

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

type LedgerSummary struct {
	orm.ModelBase                          `bson:"-" json:"-"`
	ID                                     string `bson:"_id"`
	PC                                     *ProfitCenter
	CC                                     *CostCenter
	CompanyCode                            string
	LedgerAccount                          string
	Customer                               *Customer
	Product                                *Product
	Date                                   *Date
	PLGroup1, PLGroup2, PLGroup3, PLGroup4 string
	Value1, Value2, Value3                 float64
	//EasyForSelect
	PCID, CCID, OutletID, SKUID, PLCode, PLOrder string
	Month                                        time.Month
	Year                                         int
}

// month,year
func (s *LedgerSummary) RecordID() interface{} {
	return s.ID
	//return toolkit.Sprintf("%d_%d_%s_%s", s.Date.Year, s.Date.Month, s.CompanyCode, s.LedgerAccount)
}

func (s *LedgerSummary) PrepareID() interface{} {
	s.ID = toolkit.Sprintf("%d_%d_%s_%s", s.Date.Year, s.Date.Month, s.CompanyCode, s.LedgerAccount)
	return s
}

func (s *LedgerSummary) TableName() string {
	return "ledgersummaries"
}

func GetLedgerSummaryByDetail(LedgerAccount, PCID, CCID, OutletID, SKUID string, Year int, Month time.Month) (ls *LedgerSummary) {
	ls = new(LedgerSummary)

	filter := dbox.And(dbox.Eq("month", Month),
		dbox.Eq("year", Year),
		dbox.Eq("ledgeraccount", LedgerAccount),
		dbox.Contains("pcid", PCID),
		dbox.Contains("ccid", CCID),
		dbox.Contains("outletid", OutletID),
		dbox.Contains("skuid", SKUID))

	cr, err := Find(ls, filter, nil)
	if err != nil {
		return
	}

	_ = cr.Fetch(&ls, 1, false)
	cr.Close()

	return
}

func SummaryGenerateDummyData() []*LedgerSummary {
	res := []*LedgerSummary{}
	pcs := []*ProfitCenter{}
	ccs := []*CostCenter{}
	cus := []*Customer{}
	prs := []*Product{}
	das := []*Date{}

	for i := 0; i < 5; i++ {
		pc := new(ProfitCenter)
		pc.ID = fmt.Sprintf("PC00%d", i)
		pc.EntityID = toolkit.RandomString(5)
		pc.Name = toolkit.RandomString(10)
		pc.BrandID = toolkit.RandomString(5)
		pc.BrandCategoryID = toolkit.RandomString(5)
		pc.BranchID = toolkit.RandomString(5)
		pc.BranchType = BranchTypeEnum(toolkit.RandInt(100))
		pcs = append(pcs, pc)

		cc := new(CostCenter)
		cc.ID = fmt.Sprintf("CC00%d", i)
		cc.EntityID = toolkit.RandomString(5)
		cc.Name = toolkit.RandomString(10)
		cc.CostGroup01 = toolkit.RandomString(5)
		cc.CostGroup02 = toolkit.RandomString(5)
		cc.CostGroup03 = toolkit.RandomString(5)
		cc.BranchID = toolkit.RandomString(5)
		cc.BranchType = BranchTypeEnum(toolkit.RandInt(100))
		cc.CCTypeID = toolkit.RandomString(5)
		cc.HCCGroupID = toolkit.RandomString(5)
		ccs = append(ccs, cc)

		cu := new(Customer)
		cu.ID = toolkit.RandomString(5)
		cu.BranchName = toolkit.RandomString(5)
		cu.BranchID = toolkit.RandomString(5)
		cu.Name = toolkit.RandomString(5)
		cu.KeyAccount = toolkit.RandomString(5)
		cu.ChannelName = toolkit.RandomString(5)
		cu.CustomerGroupName = toolkit.RandomString(5)
		cu.National = toolkit.RandomString(5)
		cu.Zone = toolkit.RandomString(5)
		cu.Region = toolkit.RandomString(5)
		cu.Area = toolkit.RandomString(5)
		cus = append(cus, cu)

		pr := new(Product)
		pr.ID = toolkit.RandomString(5)
		pr.Name = toolkit.RandomString(5)
		pr.Brand = toolkit.RandomString(5)
		prs = append(prs, pr)

		da := new(Date)
		da.ID = toolkit.RandomString(5)
		da.Date = time.Now()
		da.Month = time.Month(5)
		da.Quarter = toolkit.RandInt(100)
		da.Year = toolkit.RandInt(100)
		das = append(das, da)
	}

	for i := 0; i < 100; i++ {
		o := new(LedgerSummary)
		o.ID = fmt.Sprintf("LS00%d", i)
		o.PC = pcs[i%len(pcs)]
		o.CC = ccs[i%len(ccs)]
		o.CompanyCode = toolkit.RandomString(3)
		o.LedgerAccount = toolkit.RandomString(3)
		o.Customer = cus[i%len(cus)]
		o.Product = prs[i%len(prs)]
		o.Date = das[i%len(das)]
		o.Value1 = toolkit.RandFloat(3000, 2)
		o.Value2 = toolkit.RandFloat(3000, 2)
		o.Value3 = toolkit.RandFloat(3000, 2)
		o.Save()

		res = append(res, o)
	}

	return res
}

/*
[
    {_id:{col1:"D1",col2:"D2",col3:"D3"},SalesAmount:10,Qty:5,Value:2},
    {_id:{col1:"D1",col2:"D2",col3:"D4"},SalesAmount:10,Qty:3.2,Value:3},
]
row: _id.col1, _id.col2
col: _id.col3
*/
func SummarizeLedgerSum(
	filter *dbox.Filter,
	columns []string,
	datapoints []string,
	// misal: ["sum:Value1:SalesAmount","sum:Value2:Qty","avg:Value3"]
	fnTransform func(m *toolkit.M) error) ([]toolkit.M, error) {
	sum := new(LedgerSummary)
	conn := DB().Connection
	q := conn.NewQuery().From(sum.TableName())
	if filter != nil {
		q = q.Where(filter)
	}
	if len(columns) > 0 {
		cs := []string{}
		for i := range columns {
			cs = append(cs, strings.ToLower(columns[i]))
		}

		q = q.Group(cs...)
	}
	if len(datapoints) == 0 {
		return nil, errors.New("SummarizedLedgerSum: Datapoints should be defined at least 1")
	}
	for _, dp := range datapoints {
		dps := strings.Split(strings.ToLower(dp), ":")
		if len(dps) < 2 {
			return nil, errors.New("SummarizeLedgerSum: Parameters should follow this pattern aggrOp:fieldName:[alias - optional]")
		}

		fieldid := dps[1]
		alias := fieldid
		op := ""
		if !strings.HasPrefix(dps[0], "$") {
			dps[0] = "$" + strings.ToLower(dps[0])
		}

		if toolkit.HasMember([]string{dbox.AggrSum, dbox.AggrAvr, dbox.AggrMax,
			dbox.AggrMin, dbox.AggrMean, dbox.AggrMed}, dps[0]) {
			op = dps[0]
		}
		if op == "" {
			return nil, errors.New("SummarizeLedgerSum: Invalid Operation")
		}
		if len(dps) > 2 {
			alias = dps[2]
		}

		if strings.HasPrefix(alias, "$") {
			alias = alias[1:]
		}

		if fnumber, enumber := toolkit.IsStringNumber(fieldid, "."); enumber == nil {
			q = q.Aggr(op, fnumber, alias)
		} else {
			q = q.Aggr(op, fieldid, alias)
		}
	}

	c, e := q.Cursor(nil)
	if e != nil {
		return nil, errors.New("SummarizedLedgerSum: Preparing cursor error " + e.Error())
	}
	defer c.Close()

	ms := []toolkit.M{}
	e = c.Fetch(&ms, 0, false)
	if e != nil {
		return nil, errors.New("SummarizedLedgerSum: Fetch cursor error " + e.Error())
	}

	if c.Count() > 0 {
		e = c.Fetch(&ms, 0, false)
		if e != nil {
			return nil, errors.New("SummarizedLedgerSum: Fetch cursor error " + e.Error())
		}
	}

	if fnTransform != nil {
		for idx, m := range ms {
			e = fnTransform(&m)
			if e != nil {
				return nil, errors.New(toolkit.Sprintf("SummarizedLedgerSum: Transform error on index %d, %s",
					idx, e.Error()))
			}
		}
	}

	return ms, nil
}

func (s *LedgerSummary) Save() error {
	e := Save(s)
	if e != nil {
		return errors.New(toolkit.Sprintf("[%v-%v] Error found : ", s.TableName(), "save", e.Error()))
	}
	return e
}

type PivotParam struct {
	Dimensions []*PivotParamDimensions `json:"dimensions"`
	DataPoints []*PivotParamDataPoint  `json:"datapoints"`
}

type PivotParamDimensions struct {
	Field string `json:"field"`
	Type  string `json:"type"`
	Alias string `json:"alias"`
}

type PivotParamDataPoint struct {
	OP    string `json:"op"`
	Field string `json:"field"`
	Alias string `json:"alias"`
}

func (p *PivotParam) ParseDimensions() (res []string) {
	res = []string{}
	for _, each := range p.Dimensions {
		res = append(res, each.Field)
	}
	return
}

func (p *PivotParam) ParseDataPoints() (res []string) {
	for _, each := range p.DataPoints {
		parts := []string{each.OP, each.Field, each.Alias}

		if !strings.HasPrefix(parts[1], "$") {
			parts[1] = fmt.Sprintf("$%s", parts[1])
		}

		res = append(res, strings.Join(parts, ":"))
	}
	return
}

func (p *PivotParam) MapSummarizedLedger(data []toolkit.M) []toolkit.M {
	res := []toolkit.M{}
	metadata := map[string]string{}

	for i, each := range data {
		row := toolkit.M{}

		if i == 0 {
			// cache the metadata, only on first loop
			for key, val := range each {
				if key == "_id" {
					for key2 := range val.(toolkit.M) {
						keyv := key2

						for _, dimension := range p.Dimensions {
							if strings.ToLower(dimension.Field) == strings.ToLower(keyv) {
								keyv = strings.Replace(strings.Replace(dimension.Field, ".", "", -1), "_id", "_ID", -1)
							}
						}

						if key2 == "_id" {
							keyv = toolkit.TrimByString(keyv, "_")
						}

						metadata[fmt.Sprintf("%s.%s", key, key2)] = keyv
					}
				} else {
					keyv := key
					for _, each := range p.DataPoints {
						if strings.ToLower(each.Alias) == strings.ToLower(key) {
							keyv = strings.Replace(each.Alias, " ", "_", -1)
						}
					}
					metadata[key] = keyv
				}
			}
		}

		// flatten the data
		for key, val := range each {
			if key == "_id" {
				for key2, val2 := range val.(toolkit.M) {
					keyv := metadata[fmt.Sprintf("%s.%s", key, key2)]
					row.Set(keyv, val2)
				}
			} else {
				keyv := metadata[key]
				row.Set(keyv, val)
			}
		}

		res = append(res, row)
	}

	return res
}

func (p *PivotParam) GetPivotConfig(data []toolkit.M) toolkit.M {
	res := struct {
		SchemaModelFields   toolkit.M
		SchemaCubeDimension toolkit.M
		SchemaCubeMeasures  toolkit.M
		Columns             []toolkit.M
		Rows                []toolkit.M
		Measures            []string
	}{
		toolkit.M{},
		toolkit.M{},
		toolkit.M{},
		[]toolkit.M{},
		[]toolkit.M{},
		[]string{},
	}

	if len(data) > 0 {
		for key := range data[0] {
			for _, c := range p.Dimensions {
				a := strings.ToLower(strings.Replace(c.Field, ".", "", -1)) == strings.ToLower(key)
				b := strings.ToLower(toolkit.TrimByString(c.Field, "_")) == strings.ToLower(key)

				if a || b {
					if c.Type == "column" {
						res.Columns = append(res.Columns, toolkit.M{"name": key, "expand": false})
					} else {
						res.Rows = append(res.Rows, toolkit.M{"name": key, "expand": false})
					}

					caption := fmt.Sprintf("All %s", c.Alias)
					res.SchemaModelFields.Set(key, toolkit.M{"type": "string"})
					res.SchemaCubeDimension.Set(key, toolkit.M{"caption": caption})
				}
			}

			for _, c := range p.DataPoints {
				if strings.ToLower(strings.Replace(c.Alias, " ", "_", -1)) == strings.ToLower(key) {
					op := c.OP
					if op == "avg" {
						op = "average"
					}

					res.SchemaModelFields.Set(key, toolkit.M{"type": "number"})
					res.SchemaCubeMeasures.Set(key, toolkit.M{"field": key, "aggregate": op})
					res.Measures = append(res.Measures, key)
				}
			}
		}
	}

	resM, err := toolkit.ToM(res)
	if err != nil {
		fmt.Println(err.Error())
	}

	return resM
}
<commit_msg>update id for ledgersummary<commit_after>package gdrj

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
)

type LedgerSummary struct {
	orm.ModelBase                          `bson:"-" json:"-"`
	ID                                     string `bson:"_id"`
	PC                                     *ProfitCenter
	CC                                     *CostCenter
	CompanyCode                            string
	LedgerAccount                          string
	Customer                               *Customer
	Product                                *Product
	Date                                   *Date
	PLGroup1, PLGroup2, PLGroup3, PLGroup4 string
	Value1, Value2, Value3                 float64
	//EasyForSelect
	PCID, CCID, OutletID, SKUID, PLCode, PLOrder string
	Month                                        time.Month
	Year                                         int
}

// month,year
func (s *LedgerSummary) RecordID() interface{} {
	return s.ID
	//return toolkit.Sprintf("%d_%d_%s_%s", s.Date.Year, s.Date.Month, s.CompanyCode, s.LedgerAccount)
}

func (s *LedgerSummary) PrepareID() interface{} {
	s.ID = toolkit.Sprintf("%d_%d_%s_%s_%s_%s_%s_%s_%s",
		s.Date.Year, s.Date.Month,
		s.CompanyCode, s.LedgerAccount,
		s.PLCode, s.OutletID, s.SKUID, s.PCID, s.CCID)
	return s
}

func (s *LedgerSummary) TableName() string {
	return "ledgersummaries"
}

func (s *LedgerSummary) PreSave()error{
	s.ID=s.PrepareID()
	return nil
}

func GetLedgerSummaryByDetail(LedgerAccount, PCID, CCID, OutletID, SKUID string, Year int, Month time.Month) (ls *LedgerSummary) {
	ls = new(LedgerSummary)

	filter := dbox.And(dbox.Eq("month", Month),
		dbox.Eq("year", Year),
		dbox.Eq("ledgeraccount", LedgerAccount),
		dbox.Contains("pcid", PCID),
		dbox.Contains("ccid", CCID),
		dbox.Contains("outletid", OutletID),
		dbox.Contains("skuid", SKUID))

	cr, err := Find(ls, filter, nil)
	if err != nil {
		return
	}

	_ = cr.Fetch(&ls, 1, false)
	cr.Close()

	return
}

func SummaryGenerateDummyData() []*LedgerSummary {
	res := []*LedgerSummary{}
	pcs := []*ProfitCenter{}
	ccs := []*CostCenter{}
	cus := []*Customer{}
	prs := []*Product{}
	das := []*Date{}

	for i := 0; i < 5; i++ {
		pc := new(ProfitCenter)
		pc.ID = fmt.Sprintf("PC00%d", i)
		pc.EntityID = toolkit.RandomString(5)
		pc.Name = toolkit.RandomString(10)
		pc.BrandID = toolkit.RandomString(5)
		pc.BrandCategoryID = toolkit.RandomString(5)
		pc.BranchID = toolkit.RandomString(5)
		pc.BranchType = BranchTypeEnum(toolkit.RandInt(100))
		pcs = append(pcs, pc)

		cc := new(CostCenter)
		cc.ID = fmt.Sprintf("CC00%d", i)
		cc.EntityID = toolkit.RandomString(5)
		cc.Name = toolkit.RandomString(10)
		cc.CostGroup01 = toolkit.RandomString(5)
		cc.CostGroup02 = toolkit.RandomString(5)
		cc.CostGroup03 = toolkit.RandomString(5)
		cc.BranchID = toolkit.RandomString(5)
		cc.BranchType = BranchTypeEnum(toolkit.RandInt(100))
		cc.CCTypeID = toolkit.RandomString(5)
		cc.HCCGroupID = toolkit.RandomString(5)
		ccs = append(ccs, cc)

		cu := new(Customer)
		cu.ID = toolkit.RandomString(5)
		cu.BranchName = toolkit.RandomString(5)
		cu.BranchID = toolkit.RandomString(5)
		cu.Name = toolkit.RandomString(5)
		cu.KeyAccount = toolkit.RandomString(5)
		cu.ChannelName = toolkit.RandomString(5)
		cu.CustomerGroupName = toolkit.RandomString(5)
		cu.National = toolkit.RandomString(5)
		cu.Zone = toolkit.RandomString(5)
		cu.Region = toolkit.RandomString(5)
		cu.Area = toolkit.RandomString(5)
		cus = append(cus, cu)

		pr := new(Product)
		pr.ID = toolkit.RandomString(5)
		pr.Name = toolkit.RandomString(5)
		pr.Brand = toolkit.RandomString(5)
		prs = append(prs, pr)

		da := new(Date)
		da.ID = toolkit.RandomString(5)
		da.Date = time.Now()
		da.Month = time.Month(5)
		da.Quarter = toolkit.RandInt(100)
		da.Year = toolkit.RandInt(100)
		das = append(das, da)
	}

	for i := 0; i < 100; i++ {
		o := new(LedgerSummary)
		o.ID = fmt.Sprintf("LS00%d", i)
		o.PC = pcs[i%len(pcs)]
		o.CC = ccs[i%len(ccs)]
		o.CompanyCode = toolkit.RandomString(3)
		o.LedgerAccount = toolkit.RandomString(3)
		o.Customer = cus[i%len(cus)]
		o.Product = prs[i%len(prs)]
		o.Date = das[i%len(das)]
		o.Value1 = toolkit.RandFloat(3000, 2)
		o.Value2 = toolkit.RandFloat(3000, 2)
		o.Value3 = toolkit.RandFloat(3000, 2)
		o.Save()

		res = append(res, o)
	}

	return res
}

/*
[
    {_id:{col1:"D1",col2:"D2",col3:"D3"},SalesAmount:10,Qty:5,Value:2},
    {_id:{col1:"D1",col2:"D2",col3:"D4"},SalesAmount:10,Qty:3.2,Value:3},
]
row: _id.col1, _id.col2
col: _id.col3
*/
func SummarizeLedgerSum(
	filter *dbox.Filter,
	columns []string,
	datapoints []string,
	// misal: ["sum:Value1:SalesAmount","sum:Value2:Qty","avg:Value3"]
	fnTransform func(m *toolkit.M) error) ([]toolkit.M, error) {
	sum := new(LedgerSummary)
	conn := DB().Connection
	q := conn.NewQuery().From(sum.TableName())
	if filter != nil {
		q = q.Where(filter)
	}
	if len(columns) > 0 {
		cs := []string{}
		for i := range columns {
			cs = append(cs, strings.ToLower(columns[i]))
		}

		q = q.Group(cs...)
	}
	if len(datapoints) == 0 {
		return nil, errors.New("SummarizedLedgerSum: Datapoints should be defined at least 1")
	}
	for _, dp := range datapoints {
		dps := strings.Split(strings.ToLower(dp), ":")
		if len(dps) < 2 {
			return nil, errors.New("SummarizeLedgerSum: Parameters should follow this pattern aggrOp:fieldName:[alias - optional]")
		}

		fieldid := dps[1]
		alias := fieldid
		op := ""
		if !strings.HasPrefix(dps[0], "$") {
			dps[0] = "$" + strings.ToLower(dps[0])
		}

		if toolkit.HasMember([]string{dbox.AggrSum, dbox.AggrAvr, dbox.AggrMax,
			dbox.AggrMin, dbox.AggrMean, dbox.AggrMed}, dps[0]) {
			op = dps[0]
		}
		if op == "" {
			return nil, errors.New("SummarizeLedgerSum: Invalid Operation")
		}
		if len(dps) > 2 {
			alias = dps[2]
		}

		if strings.HasPrefix(alias, "$") {
			alias = alias[1:]
		}

		if fnumber, enumber := toolkit.IsStringNumber(fieldid, "."); enumber == nil {
			q = q.Aggr(op, fnumber, alias)
		} else {
			q = q.Aggr(op, fieldid, alias)
		}
	}

	c, e := q.Cursor(nil)
	if e != nil {
		return nil, errors.New("SummarizedLedgerSum: Preparing cursor error " + e.Error())
	}
	defer c.Close()

	ms := []toolkit.M{}
	e = c.Fetch(&ms, 0, false)
	if e != nil {
		return nil, errors.New("SummarizedLedgerSum: Fetch cursor error " + e.Error())
	}

	if c.Count() > 0 {
		e = c.Fetch(&ms, 0, false)
		if e != nil {
			return nil, errors.New("SummarizedLedgerSum: Fetch cursor error " + e.Error())
		}
	}

	if fnTransform != nil {
		for idx, m := range ms {
			e = fnTransform(&m)
			if e != nil {
				return nil, errors.New(toolkit.Sprintf("SummarizedLedgerSum: Transform error on index %d, %s",
					idx, e.Error()))
			}
		}
	}

	return ms, nil
}

func (s *LedgerSummary) Save() error {
	e := Save(s)
	if e != nil {
		return errors.New(toolkit.Sprintf("[%v-%v] Error found : ", s.TableName(), "save", e.Error()))
	}
	return e
}

type PivotParam struct {
	Dimensions []*PivotParamDimensions `json:"dimensions"`
	DataPoints []*PivotParamDataPoint  `json:"datapoints"`
}

type PivotParamDimensions struct {
	Field string `json:"field"`
	Type  string `json:"type"`
	Alias string `json:"alias"`
}

type PivotParamDataPoint struct {
	OP    string `json:"op"`
	Field string `json:"field"`
	Alias string `json:"alias"`
}

func (p *PivotParam) ParseDimensions() (res []string) {
	res = []string{}
	for _, each := range p.Dimensions {
		res = append(res, each.Field)
	}
	return
}

func (p *PivotParam) ParseDataPoints() (res []string) {
	for _, each := range p.DataPoints {
		parts := []string{each.OP, each.Field, each.Alias}

		if !strings.HasPrefix(parts[1], "$") {
			parts[1] = fmt.Sprintf("$%s", parts[1])
		}

		res = append(res, strings.Join(parts, ":"))
	}
	return
}

func (p *PivotParam) MapSummarizedLedger(data []toolkit.M) []toolkit.M {
	res := []toolkit.M{}
	metadata := map[string]string{}

	for i, each := range data {
		row := toolkit.M{}

		if i == 0 {
			// cache the metadata, only on first loop
			for key, val := range each {
				if key == "_id" {
					for key2 := range val.(toolkit.M) {
						keyv := key2

						for _, dimension := range p.Dimensions {
							if strings.ToLower(dimension.Field) == strings.ToLower(keyv) {
								keyv = strings.Replace(strings.Replace(dimension.Field, ".", "", -1), "_id", "_ID", -1)
							}
						}

						if key2 == "_id" {
							keyv = toolkit.TrimByString(keyv, "_")
						}

						metadata[fmt.Sprintf("%s.%s", key, key2)] = keyv
					}
				} else {
					keyv := key
					for _, each := range p.DataPoints {
						if strings.ToLower(each.Alias) == strings.ToLower(key) {
							keyv = strings.Replace(each.Alias, " ", "_", -1)
						}
					}
					metadata[key] = keyv
				}
			}
		}

		// flatten the data
		for key, val := range each {
			if key == "_id" {
				for key2, val2 := range val.(toolkit.M) {
					keyv := metadata[fmt.Sprintf("%s.%s", key, key2)]
					row.Set(keyv, val2)
				}
			} else {
				keyv := metadata[key]
				row.Set(keyv, val)
			}
		}

		res = append(res, row)
	}

	return res
}

func (p *PivotParam) GetPivotConfig(data []toolkit.M) toolkit.M {
	res := struct {
		SchemaModelFields   toolkit.M
		SchemaCubeDimension toolkit.M
		SchemaCubeMeasures  toolkit.M
		Columns             []toolkit.M
		Rows                []toolkit.M
		Measures            []string
	}{
		toolkit.M{},
		toolkit.M{},
		toolkit.M{},
		[]toolkit.M{},
		[]toolkit.M{},
		[]string{},
	}

	if len(data) > 0 {
		for key := range data[0] {
			for _, c := range p.Dimensions {
				a := strings.ToLower(strings.Replace(c.Field, ".", "", -1)) == strings.ToLower(key)
				b := strings.ToLower(toolkit.TrimByString(c.Field, "_")) == strings.ToLower(key)

				if a || b {
					if c.Type == "column" {
						res.Columns = append(res.Columns, toolkit.M{"name": key, "expand": false})
					} else {
						res.Rows = append(res.Rows, toolkit.M{"name": key, "expand": false})
					}

					caption := fmt.Sprintf("All %s", c.Alias)
					res.SchemaModelFields.Set(key, toolkit.M{"type": "string"})
					res.SchemaCubeDimension.Set(key, toolkit.M{"caption": caption})
				}
			}

			for _, c := range p.DataPoints {
				if strings.ToLower(strings.Replace(c.Alias, " ", "_", -1)) == strings.ToLower(key) {
					op := c.OP
					if op == "avg" {
						op = "average"
					}

					res.SchemaModelFields.Set(key, toolkit.M{"type": "number"})
					res.SchemaCubeMeasures.Set(key, toolkit.M{"field": key, "aggregate": op})
					res.Measures = append(res.Measures, key)
				}
			}
		}
	}

	resM, err := toolkit.ToM(res)
	if err != nil {
		fmt.Println(err.Error())
	}

	return resM
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"path/filepath"
	"regexp"
	"strings"
)

type defines map[string][]string

var (
	macroRegexp    = regexp.MustCompile(`\$\{\/?[^@]*?\}`)
	embMacroRegexp = regexp.MustCompile(`\$\{\/?@\}`)
)

// set устанавливает значение макроопределения или добавляет новое.
func (def defines) set(name, val string) {
	def[name] = []string{val}
}

// Раскручивает определения, делая всевозможные подстановки.
func (def defines) bootstrap() {
	// для каждого макроопределения
	for name, values := range def {
		res := make([]string, 0, 64)
		// для каждого значения в макроопределении
		for _, val := range values {
			// выполнить подстановку макровызовов
			newVal := def.substitute([]string{val}, macroRegexp)
			res = append(res, newVal...)
		}
		def[name] = res
	}
}

func substituteEmbDefs(input, sources []string) []string {
	emb := defines{"@": sources}
	return emb.substituteDefs(input, embMacroRegexp)
}

func (def defines) substituteUserDefs(input []string) []string {
	return def.substituteDefs(input, macroRegexp)
}

func (def defines) substituteDefs(input []string, r *regexp.Regexp) []string {
	res := make([]string, 0, 64)
	for _, val := range input {
		res = append(res, def.substitute([]string{val}, r)...)
	}
	return res
}

// Делает подстановки определений на места макровызовов.
// input должен содержать одинаковые макровызовы!
// FIXME: сделать проверку зацикливания.
func (def defines) substitute(input []string, re *regexp.Regexp) []string {
	cached := input

	for {
		// выходные значения на каждой итерации
		result := make([]string, 0, 16)

		for _, str := range input {
			// поиск макровызова
			macroCall := re.FindString(str)

			if len(macroCall) == 0 {
				result = append(result, str)

				if cached[0] != input[0] {
					log.Printf("macro: %s -> %v", cached, input)
				}

				return input
			}

			// извлечение имени
			name := macroCall[2 : len(macroCall)-1]

			// проверка префикса
			basePath := false
			if name[0] == '/' {
				basePath = true
				name = name[1:]
			}

			// поиск значения
			values, exists := def[name]
			if !exists {
				throw("Macro definition with name %s not found", name)
			}

			// применение модификатора
			if basePath {
				values = basePathModif(values)
			}

			// подстановка каждым значением макроопределения
			for _, val := range values {
				subs := strings.Replace(str, macroCall, val, 1)
				result = append(result, subs)
			}
		}

		input = result
	}

	return input
}

// Модификатор значений переменной - базовый путь.
func basePathModif(v []string) []string {
	short := make([]string, len(v))
	for i, _ := range v {
		short[i] = filepath.Base(v[i])
	}
	return short
}
<commit_msg>Изменен формат макроподстановок с ${} на $()<commit_after>package main

import (
	"log"
	"path/filepath"
	"regexp"
	"strings"
)

type defines map[string][]string

var (
	macroRegexp    = regexp.MustCompile(`\$\(\/?[^@]*?\)`)
	embMacroRegexp = regexp.MustCompile(`\$\(\/?@\)`)
)

// set устанавливает значение макроопределения или добавляет новое.
func (def defines) set(name, val string) {
	def[name] = []string{val}
}

// Раскручивает определения, делая всевозможные подстановки.
func (def defines) bootstrap() {
	// для каждого макроопределения
	for name, values := range def {
		res := make([]string, 0, 64)
		// для каждого значения в макроопределении
		for _, val := range values {
			// выполнить подстановку макровызовов
			newVal := def.substitute([]string{val}, macroRegexp)
			res = append(res, newVal...)
		}
		def[name] = res
	}
}

func substituteEmbDefs(input, sources []string) []string {
	emb := defines{"@": sources}
	return emb.substituteDefs(input, embMacroRegexp)
}

func (def defines) substituteUserDefs(input []string) []string {
	return def.substituteDefs(input, macroRegexp)
}

func (def defines) substituteDefs(input []string, r *regexp.Regexp) []string {
	res := make([]string, 0, 64)
	for _, val := range input {
		res = append(res, def.substitute([]string{val}, r)...)
	}
	return res
}

// Делает подстановки определений на места макровызовов.
// input должен содержать одинаковые макровызовы!
// FIXME: сделать проверку зацикливания.
func (def defines) substitute(input []string, re *regexp.Regexp) []string {
	cached := input

	for {
		// выходные значения на каждой итерации
		result := make([]string, 0, 16)

		for _, str := range input {
			// поиск макровызова
			macroCall := re.FindString(str)

			if len(macroCall) == 0 {
				result = append(result, str)

				if cached[0] != input[0] {
					log.Printf("macro: %s -> %v", cached, input)
				}

				return input
			}

			// извлечение имени
			name := macroCall[2 : len(macroCall)-1]

			// проверка префикса
			basePath := false
			if name[0] == '/' {
				basePath = true
				name = name[1:]
			}

			// поиск значения
			values, exists := def[name]
			if !exists {
				throw("Macro definition with name %s not found", name)
			}

			// применение модификатора
			if basePath {
				values = basePathModif(values)
			}

			// подстановка каждым значением макроопределения
			for _, val := range values {
				subs := strings.Replace(str, macroCall, val, 1)
				result = append(result, subs)
			}
		}

		input = result
	}

	return input
}

// Модификатор значений переменной - базовый путь.
func basePathModif(v []string) []string {
	short := make([]string, len(v))
	for i, _ := range v {
		short[i] = filepath.Base(v[i])
	}
	return short
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	dsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/nqm_parser"
	"github.com/Cepave/open-falcon-backend/modules/query/g"
	"github.com/Cepave/open-falcon-backend/modules/query/nqm"
	log "github.com/Sirupsen/logrus"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/context"
	"github.com/bitly/go-simplejson"
	"net/http"
	"runtime/debug"
	"strconv"
	"strings"
	"time"
)

const (
	jsonIndent = false
	jsonCoding = false
)

var nqmService nqm.ServiceController

func configNqmRoutes() {
	nqmService = nqm.GetDefaultServiceController()
	nqmService.Init()

	/**
	 * Registers the handler of RESTful service on beego
	 */
	serviceController := beego.NewControllerRegister()
	setupUrlMappingAndHandler(serviceController)
	// :~)

	http.Handle("/nqm/", serviceController)
}

func setupUrlMappingAndHandler(serviceRegister *beego.ControllerRegister) {
	serviceRegister.AddMethod(
		"get", "/nqm/icmp/list/by-provinces",
		listIcmpByProvinces,
	)
	serviceRegister.AddMethod(
		"get", "/nqm/icmp/province/:province_id([0-9]+)/list/by-targets",
		listIcmpByTargetsForAProvince,
	)
	serviceRegister.AddMethod(
		"get", "/nqm/province/:province_id([0-9]+)/agents",
		listAgentsInProvince,
	)
}

type resultWithDsl struct {
	queryParams *dsl.QueryParams
	resultData  interface{}
}

func (result *resultWithDsl) MarshalJSON() ([]byte, error) {
	jsonObject := simplejson.New()

	jsonObject.SetPath([]string{"dsl", "start_time"}, result.queryParams.StartTime.Unix())
	jsonObject.SetPath([]string{"dsl", "end_time"}, result.queryParams.EndTime.Unix())
	jsonObject.Set("result", result.resultData)

	return jsonObject.MarshalJSON()
}

// Lists agents(grouped by city) for a province
func listAgentsInProvince(ctx *context.Context) {
	provinceId, _ := strconv.ParseInt(ctx.Input.Param(":province_id"), 10, 16)
	ctx.Output.JSON(nqm.ListAgentsInCityByProvinceId(int32(provinceId)), jsonIndent, jsonCoding)
}

// Lists statistics data of ICMP, which would be grouped by provinces
func listIcmpByProvinces(ctx *context.Context) {
	defer outputJsonForPanic(ctx)

	dslParams, isValid := processDslAndOutputError(ctx, ctx.Input.Query("dsl"))
	if !isValid {
		return
	}

	listResult := nqmService.ListByProvinces(dslParams)

	ctx.Output.JSON(&resultWithDsl{queryParams: dslParams, resultData: listResult}, jsonIndent, jsonCoding)
}

// Lists data of targets, which would be grouped by cities
func listIcmpByTargetsForAProvince(ctx *context.Context) {
	defer outputJsonForPanic(ctx)

	dslParams, isValid := processDslAndOutputError(ctx, ctx.Input.Query("dsl"))
	if !isValid {
		return
	}

	dslParams.AgentFilter.MatchProvinces = make([]string, 0) // Ignores the province of agent

	provinceId, _ := strconv.ParseInt(ctx.Input.Param(":province_id"), 10, 16)
	dslParams.AgentFilterById.MatchProvinces = []int16{int16(provinceId)} // Use the id as the filter of agent

	if agentId, parseErrForAgentId := strconv.ParseInt(ctx.Input.Query("agent_id"), 10, 16); parseErrForAgentId == nil {
		dslParams.AgentFilterById.MatchIds = []int32{int32(agentId)} // Set the filter by agent's id
	} else if cityId, parseErrForCityId := strconv.ParseInt(ctx.Input.Query("city_id_of_agent"), 10, 16); parseErrForCityId == nil {
		dslParams.AgentFilterById.MatchCities = []int16{int16(cityId)} // Set the filter by city's id
	}

	listResult := nqmService.ListTargetsWithCityDetail(dslParams)
	ctx.Output.JSON(&resultWithDsl{queryParams: dslParams, resultData: listResult}, jsonIndent, jsonCoding)
}

type jsonDslError struct {
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func outputDslError(ctx *context.Context, err error) {
	ctx.Output.SetStatus(http.StatusBadRequest)
	ctx.Output.JSON(
		&jsonDslError{
			Code:    1,
			Message: err.Error(),
		}, jsonIndent, jsonCoding,
	)
}

// Used to output JSON message even if the execution is panic
func outputJsonForPanic(ctx *context.Context) {
	r := recover()
	if r == nil {
		return
	}

	if g.Config().Debug {
		debug.PrintStack()
	}

	log.Printf("Error on HTTP Request[%v/%v]. Error: %v", ctx.Input.Method(), ctx.Input.URI(), r)

	ctx.Output.SetStatus(http.StatusBadRequest)
	ctx.Output.JSON(&jsonDslError{
		Code:    -1,
		Message: fmt.Sprintf("%v", r),
	}, jsonIndent, jsonCoding)
}

const (
	defaultDaysForTimeRange = 7
	after7Days              = defaultDaysForTimeRange * 24 * time.Hour
	before7Days             = after7Days * -1
)

// Process DSL and output error
// Returns: true if the DSL is valid
func processDslAndOutputError(ctx *context.Context, dslText string) (*dsl.QueryParams, bool) {
	dslParams, err := processDsl(dslText)

	if err != nil {
		outputDslError(ctx, err)
		return nil, false
	}

	return dslParams, true
}

// The query of DSL would be inner-province(used for phase 1)
func processDsl(dslParams string) (*dsl.QueryParams, error) {
	strNqmDsl := strings.TrimSpace(dslParams)

	/**
	 * If any of errors for parsing DSL
	 */
	result, parseError := dsl.Parse(
		"Query.nqmdsl", []byte(strNqmDsl),
	)
	if parseError != nil {
		return nil, parseError
	}
	// :~)

	resultDsl := result.(*dsl.QueryParams)

	setupTimeRange(resultDsl)
	setupInnerProvince(resultDsl)

	paramsError := resultDsl.CheckRationalOfParameters()
	if paramsError != nil {
		return nil, paramsError
	}

	return resultDsl, nil
}

// Sets-up the time range with provided-or-not value of parameters
// 1. Without any parameter of time range
// 2. Has only start time
// 3. Has only end time
func setupTimeRange(queryParams *dsl.QueryParams) {
	if queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		now := time.Now()

		queryParams.StartTime = now.Add(before7Days)  // Include 7 days before
		queryParams.EndTime = now.Add(24 * time.Hour) // Include today
		return
	}

	if queryParams.StartTime.IsZero() && !queryParams.EndTime.IsZero() {
		queryParams.StartTime = queryParams.EndTime.Add(before7Days)
		return
	}

	if !queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		queryParams.EndTime = queryParams.StartTime.Add(after7Days)
		return
	}

	if queryParams.StartTime.Unix() == queryParams.EndTime.Unix() {
		queryParams.EndTime = queryParams.StartTime.Add(24 * time.Hour)
	}
}

/**
 * !IMPORTANT!
 * This default value is just used in phase 1 funcion of NQM reporting(inner-province)
 */
func setupInnerProvince(queryParams *dsl.QueryParams) {
	queryParams.ProvinceRelation = dsl.SAME_VALUE
}
<commit_msg>Fix CORS for Restful services of NQM<commit_after>package http

import (
	"fmt"
	dsl "github.com/Cepave/open-falcon-backend/modules/query/dsl/nqm_parser"
	"github.com/Cepave/open-falcon-backend/modules/query/g"
	"github.com/Cepave/open-falcon-backend/modules/query/nqm"
	log "github.com/Sirupsen/logrus"
	"github.com/astaxie/beego/plugins/cors"
	"github.com/astaxie/beego"
	"github.com/astaxie/beego/context"
	"github.com/bitly/go-simplejson"
	"net/http"
	"runtime/debug"
	"strconv"
	"strings"
	"time"
)

const (
	jsonIndent = false
	jsonCoding = false
)

var nqmService nqm.ServiceController

func configNqmRoutes() {
	nqmService = nqm.GetDefaultServiceController()
	nqmService.Init()

	/**
	 * Registers the handler of RESTful service on beego
	 */
	serviceController := beego.NewControllerRegister()
	serviceController.InsertFilter(
		"*",
		beego.BeforeRouter,
		cors.Allow(&cors.Options{
			AllowAllOrigins: true,
		}),
	)
	setupUrlMappingAndHandler(serviceController)
	// :~)

	http.Handle("/nqm/", serviceController)
}

func setupUrlMappingAndHandler(serviceRegister *beego.ControllerRegister) {
	serviceRegister.AddMethod(
		"get", "/nqm/icmp/list/by-provinces",
		listIcmpByProvinces,
	)
	serviceRegister.AddMethod(
		"get", "/nqm/icmp/province/:province_id([0-9]+)/list/by-targets",
		listIcmpByTargetsForAProvince,
	)
	serviceRegister.AddMethod(
		"get", "/nqm/province/:province_id([0-9]+)/agents",
		listAgentsInProvince,
	)
}

type resultWithDsl struct {
	queryParams *dsl.QueryParams
	resultData  interface{}
}

func (result *resultWithDsl) MarshalJSON() ([]byte, error) {
	jsonObject := simplejson.New()

	jsonObject.SetPath([]string{"dsl", "start_time"}, result.queryParams.StartTime.Unix())
	jsonObject.SetPath([]string{"dsl", "end_time"}, result.queryParams.EndTime.Unix())
	jsonObject.Set("result", result.resultData)

	return jsonObject.MarshalJSON()
}

// Lists agents(grouped by city) for a province
func listAgentsInProvince(ctx *context.Context) {
	provinceId, _ := strconv.ParseInt(ctx.Input.Param(":province_id"), 10, 16)
	ctx.Output.JSON(nqm.ListAgentsInCityByProvinceId(int32(provinceId)), jsonIndent, jsonCoding)
}

// Lists statistics data of ICMP, which would be grouped by provinces
func listIcmpByProvinces(ctx *context.Context) {
	defer outputJsonForPanic(ctx)

	dslParams, isValid := processDslAndOutputError(ctx, ctx.Input.Query("dsl"))
	if !isValid {
		return
	}

	listResult := nqmService.ListByProvinces(dslParams)

	ctx.Output.JSON(&resultWithDsl{queryParams: dslParams, resultData: listResult}, jsonIndent, jsonCoding)
}

// Lists data of targets, which would be grouped by cities
func listIcmpByTargetsForAProvince(ctx *context.Context) {
	defer outputJsonForPanic(ctx)

	dslParams, isValid := processDslAndOutputError(ctx, ctx.Input.Query("dsl"))
	if !isValid {
		return
	}

	dslParams.AgentFilter.MatchProvinces = make([]string, 0) // Ignores the province of agent

	provinceId, _ := strconv.ParseInt(ctx.Input.Param(":province_id"), 10, 16)
	dslParams.AgentFilterById.MatchProvinces = []int16{int16(provinceId)} // Use the id as the filter of agent

	if agentId, parseErrForAgentId := strconv.ParseInt(ctx.Input.Query("agent_id"), 10, 16); parseErrForAgentId == nil {
		dslParams.AgentFilterById.MatchIds = []int32{int32(agentId)} // Set the filter by agent's id
	} else if cityId, parseErrForCityId := strconv.ParseInt(ctx.Input.Query("city_id_of_agent"), 10, 16); parseErrForCityId == nil {
		dslParams.AgentFilterById.MatchCities = []int16{int16(cityId)} // Set the filter by city's id
	}

	listResult := nqmService.ListTargetsWithCityDetail(dslParams)
	ctx.Output.JSON(&resultWithDsl{queryParams: dslParams, resultData: listResult}, jsonIndent, jsonCoding)
}

type jsonDslError struct {
	Code    int    `json:"error_code"`
	Message string `json:"error_message"`
}

func outputDslError(ctx *context.Context, err error) {
	ctx.Output.SetStatus(http.StatusBadRequest)
	ctx.Output.JSON(
		&jsonDslError{
			Code:    1,
			Message: err.Error(),
		}, jsonIndent, jsonCoding,
	)
}

// Used to output JSON message even if the execution is panic
func outputJsonForPanic(ctx *context.Context) {
	r := recover()
	if r == nil {
		return
	}

	if g.Config().Debug {
		debug.PrintStack()
	}

	log.Printf("Error on HTTP Request[%v/%v]. Error: %v", ctx.Input.Method(), ctx.Input.URI(), r)

	ctx.Output.SetStatus(http.StatusBadRequest)
	ctx.Output.JSON(&jsonDslError{
		Code:    -1,
		Message: fmt.Sprintf("%v", r),
	}, jsonIndent, jsonCoding)
}

const (
	defaultDaysForTimeRange = 7
	after7Days              = defaultDaysForTimeRange * 24 * time.Hour
	before7Days             = after7Days * -1
)

// Process DSL and output error
// Returns: true if the DSL is valid
func processDslAndOutputError(ctx *context.Context, dslText string) (*dsl.QueryParams, bool) {
	dslParams, err := processDsl(dslText)

	if err != nil {
		outputDslError(ctx, err)
		return nil, false
	}

	return dslParams, true
}

// The query of DSL would be inner-province(used for phase 1)
func processDsl(dslParams string) (*dsl.QueryParams, error) {
	strNqmDsl := strings.TrimSpace(dslParams)

	/**
	 * If any of errors for parsing DSL
	 */
	result, parseError := dsl.Parse(
		"Query.nqmdsl", []byte(strNqmDsl),
	)
	if parseError != nil {
		return nil, parseError
	}
	// :~)

	resultDsl := result.(*dsl.QueryParams)

	setupTimeRange(resultDsl)
	setupInnerProvince(resultDsl)

	paramsError := resultDsl.CheckRationalOfParameters()
	if paramsError != nil {
		return nil, paramsError
	}

	return resultDsl, nil
}

// Sets-up the time range with provided-or-not value of parameters
// 1. Without any parameter of time range
// 2. Has only start time
// 3. Has only end time
func setupTimeRange(queryParams *dsl.QueryParams) {
	if queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		now := time.Now()

		queryParams.StartTime = now.Add(before7Days)  // Include 7 days before
		queryParams.EndTime = now.Add(24 * time.Hour) // Include today
		return
	}

	if queryParams.StartTime.IsZero() && !queryParams.EndTime.IsZero() {
		queryParams.StartTime = queryParams.EndTime.Add(before7Days)
		return
	}

	if !queryParams.StartTime.IsZero() && queryParams.EndTime.IsZero() {
		queryParams.EndTime = queryParams.StartTime.Add(after7Days)
		return
	}

	if queryParams.StartTime.Unix() == queryParams.EndTime.Unix() {
		queryParams.EndTime = queryParams.StartTime.Add(24 * time.Hour)
	}
}

/**
 * !IMPORTANT!
 * This default value is just used in phase 1 funcion of NQM reporting(inner-province)
 */
func setupInnerProvince(queryParams *dsl.QueryParams) {
	queryParams.ProvinceRelation = dsl.SAME_VALUE
}
<|endoftext|>


<commit_before>package workers_test

import (
	//"github.com/APTrust/exchange/workers"
	//"github.com/stretchr/testify/assert"
	//"github.com/stretchr/testify/require"
	//"net/http"
	//"net/http/httptest"
	"testing"
)

func TestNewGlacierRestore(t *testing.T) {

}

func TestGetGlacierRestoreState(t *testing.T) {

}

func TestHandleMessage(t *testing.T) {

}

func TestRequestObject(t *testing.T) {

}

func TestRestoreRequestNeeded(t *testing.T) {

}

func TestGetS3HeadClient(t *testing.T) {

}

func TestGetIntellectualObject(t *testing.T) {

}

func TestGetGenericFile(t *testing.T) {

}

func TestUpdateWorkItem(t *testing.T) {

}

func TestSaveWorkItemState(t *testing.T) {

}

func TestFinishWithError(t *testing.T) {

}

func TestRequeueForAdditionalRequests(t *testing.T) {

}

func TestRequeueToCheckState(t *testing.T) {

}

func TestCreateRestoreWorkItem(t *testing.T) {

}

func TestRequestAllFiles(t *testing.T) {

}

func TestRequestFile(t *testing.T) {

}

func TestGetRequestDetails(t *testing.T) {

}

func TestGetRequestRecord(t *testing.T) {

}

func TestInitializeRetrieval(t *testing.T) {

}

// -------------------------------------------------------------------------
// TODO: End-to-end test with the following:
//
// 1. IntellectualObject where all requests succeed.
// 2. IntellectualObject where some requests do not succeed.
//    This should be requeued for retry.
// 3. GenericFile where request succeeds.
// 4. GenericFile where request fails (and is retried).
//
// TODO: Mocks for the following...
//
// 1. Glacier restore request
// 2. S3 head request
// 3. NSQ requeue
//
// Will need a customized Context object where URLs for NSQ,
// Pharos, S3, and Glacier point to the mock services.
// -------------------------------------------------------------------------
<commit_msg>Starting restore worker tests<commit_after>package workers_test

import (
	"github.com/APTrust/exchange/util/testutil"
	"github.com/APTrust/exchange/workers"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	//"net/http"
	//"net/http/httptest"
	"testing"
)

func getGlacierRestoreWorker(t *testing.T) *workers.APTGlacierRestoreInit {
	_context, err := testutil.GetContext("integration.json")
	require.Nil(t, err)
	return workers.NewGlacierRestore(_context)
}

func TestNewGlacierRestore(t *testing.T) {
	glacierRestore := getGlacierRestoreWorker(t)
	require.NotNil(t, glacierRestore)
	assert.NotNil(t, glacierRestore.Context)
	assert.NotNil(t, glacierRestore.RequestChannel)
	assert.NotNil(t, glacierRestore.CleanupChannel)
}

func TestGetGlacierRestoreState(t *testing.T) {

}

func TestHandleMessage(t *testing.T) {

}

func TestRequestObject(t *testing.T) {

}

func TestRestoreRequestNeeded(t *testing.T) {

}

func TestGetS3HeadClient(t *testing.T) {

}

func TestGetIntellectualObject(t *testing.T) {

}

func TestGetGenericFile(t *testing.T) {

}

func TestUpdateWorkItem(t *testing.T) {

}

func TestSaveWorkItemState(t *testing.T) {

}

func TestFinishWithError(t *testing.T) {

}

func TestRequeueForAdditionalRequests(t *testing.T) {

}

func TestRequeueToCheckState(t *testing.T) {

}

func TestCreateRestoreWorkItem(t *testing.T) {

}

func TestRequestAllFiles(t *testing.T) {

}

func TestRequestFile(t *testing.T) {

}

func TestGetRequestDetails(t *testing.T) {

}

func TestGetRequestRecord(t *testing.T) {

}

func TestInitializeRetrieval(t *testing.T) {

}

// -------------------------------------------------------------------------
// TODO: End-to-end test with the following:
//
// 1. IntellectualObject where all requests succeed.
// 2. IntellectualObject where some requests do not succeed.
//    This should be requeued for retry.
// 3. GenericFile where request succeeds.
// 4. GenericFile where request fails (and is retried).
//
// TODO: Mocks for the following...
//
// 1. Glacier restore request
// 2. S3 head request
// 3. NSQ requeue
//
// Will need a customized Context object where URLs for NSQ,
// Pharos, S3, and Glacier point to the mock services.
// -------------------------------------------------------------------------
<|endoftext|>


<commit_before>package renter

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strconv"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

const (
	PersistFilename = "renter.json"
	ShareExtension  = ".sia"
	logFile         = modules.RenterDir + ".log"
)

var (
	ErrNoNicknames    = errors.New("at least one nickname must be supplied")
	ErrNonShareSuffix = errors.New("suffix of file must be " + ShareExtension)
	ErrBadFile        = errors.New("not a .sia file")
	ErrIncompatible   = errors.New("file is not compatible with current version")

	shareHeader  = [15]byte{'S', 'i', 'a', ' ', 'S', 'h', 'a', 'r', 'e', 'd', ' ', 'F', 'i', 'l', 'e'}
	shareVersion = "0.4"

	saveMetadata = persist.Metadata{
		Header:  "Renter Persistence",
		Version: "0.4",
	}
)

// MarshalSia implements the encoding.SiaMarshaller interface, writing the
// file data to w.
func (f *file) MarshalSia(w io.Writer) error {
	enc := encoding.NewEncoder(w)

	// encode easy fields
	err := enc.EncodeAll(
		f.name,
		f.size,
		f.masterKey,
		f.pieceSize,
		f.mode,
	)
	if err != nil {
		return err
	}
	// COMPATv0.4.3 - encode the bytesUploaded and chunksUploaded fields
	// TODO: the resulting .sia file may confuse old clients.
	err = enc.EncodeAll(f.pieceSize*f.numChunks()*uint64(f.erasureCode.NumPieces()), f.numChunks())
	if err != nil {
		return err
	}

	// encode erasureCode
	switch code := f.erasureCode.(type) {
	case *rsCode:
		err = enc.EncodeAll(
			"Reed-Solomon",
			uint64(code.dataPieces),
			uint64(code.numPieces-code.dataPieces),
		)
		if err != nil {
			return err
		}
	default:
		if build.DEBUG {
			panic("unknown erasure code")
		}
		return errors.New("unknown erasure code")
	}
	// encode contracts
	if err := enc.Encode(uint64(len(f.contracts))); err != nil {
		return err
	}
	for _, c := range f.contracts {
		if err := enc.Encode(c); err != nil {
			return err
		}
	}
	return nil
}

// UnmarshalSia implements the encoding.SiaUnmarshaller interface,
// reconstructing a file from the encoded bytes read from r.
func (f *file) UnmarshalSia(r io.Reader) error {
	dec := encoding.NewDecoder(r)

	// COMPATv0.4.3 - decode bytesUploaded and chunksUploaded into dummy vars.
	var bytesUploaded, chunksUploaded uint64

	// decode easy fields
	err := dec.DecodeAll(
		&f.name,
		&f.size,
		&f.masterKey,
		&f.pieceSize,
		&f.mode,
		&bytesUploaded,
		&chunksUploaded,
	)
	if err != nil {
		return err
	}

	// decode erasure coder
	var codeType string
	if err := dec.Decode(&codeType); err != nil {
		return err
	}
	switch codeType {
	case "Reed-Solomon":
		var nData, nParity uint64
		err = dec.DecodeAll(
			&nData,
			&nParity,
		)
		if err != nil {
			return err
		}
		rsc, err := NewRSCode(int(nData), int(nParity))
		if err != nil {
			return err
		}
		f.erasureCode = rsc
	default:
		return errors.New("unrecognized erasure code type: " + codeType)
	}

	// decode contracts
	var nContracts uint64
	if err := dec.Decode(&nContracts); err != nil {
		return err
	}
	f.contracts = make(map[types.FileContractID]fileContract)
	var contract fileContract
	for i := uint64(0); i < nContracts; i++ {
		if err := dec.Decode(&contract); err != nil {
			return err
		}
		f.contracts[contract.ID] = contract
	}
	return nil
}

// saveFile saves a file to the renter directory.
func (r *Renter) saveFile(f *file) error {
	// Create directory structure specified in nickname.
	fullPath := filepath.Join(r.persistDir, f.name+ShareExtension)
	err := os.MkdirAll(filepath.Dir(fullPath), 0700)
	if err != nil {
		return err
	}

	// Open SafeFile handle.
	handle, err := persist.NewSafeFile(filepath.Join(r.persistDir, f.name+ShareExtension))
	if err != nil {
		return err
	}
	defer handle.Close()

	// Write file data.
	err = shareFiles([]*file{f}, handle)
	if err != nil {
		return err
	}

	// Commit the SafeFile.
	return handle.Commit()
}

// save stores the current renter data to disk.
func (r *Renter) save() error {
	data := struct {
		Tracking map[string]trackedFile
	}{r.tracking}
	return persist.SaveFile(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename))
}

// saveSync stores the current renter data to disk and then syncs to disk.
func (r *Renter) saveSync() error {
	data := struct {
		Tracking map[string]trackedFile
	}{r.tracking}
	return persist.SaveFileSync(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename))
}

// load fetches the saved renter data from disk.
func (r *Renter) load() error {
	// Recursively load all files found in renter directory. Errors
	// encountered during loading are logged, but are not considered fatal.
	err := filepath.Walk(r.persistDir, func(path string, info os.FileInfo, err error) error {
		// This error is non-nil if filepath.Walk couldn't stat a file or
		// folder. This generally indicates a serious error.
		if err != nil {
			return err
		}

		// Skip folders and non-sia files.
		if info.IsDir() || filepath.Ext(path) != ShareExtension {
			return nil
		}

		// Open the file.
		file, err := os.Open(path)
		if err != nil {
			r.log.Println("ERROR: could not open .sia file:", err)
			return nil
		}
		defer file.Close()

		// Load the file contents into the renter.
		_, err = r.loadSharedFiles(file)
		if err != nil {
			r.log.Println("ERROR: could not load .sia file:", err)
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Load contracts, repair set, and entropy.
	data := struct {
		Tracking  map[string]trackedFile
		Repairing map[string]string // COMPATv0.4.8
	}{}
	err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.persistDir, PersistFilename))
	if err != nil {
		return err
	}
	if data.Tracking != nil {
		r.tracking = data.Tracking
	}

	return nil
}

// shareFiles writes the specified files to w. First a header is written,
// followed by the gzipped concatenation of each file.
func shareFiles(files []*file, w io.Writer) error {
	// Write header.
	err := encoding.NewEncoder(w).EncodeAll(
		shareHeader,
		shareVersion,
		uint64(len(files)),
	)
	if err != nil {
		return err
	}

	// Create compressor.
	zip, _ := gzip.NewWriterLevel(w, gzip.BestCompression)
	enc := encoding.NewEncoder(zip)

	// Encode each file.
	for _, f := range files {
		err = enc.Encode(f)
		if err != nil {
			return err
		}
	}

	return zip.Close()
}

// ShareFile saves the specified files to shareDest.
func (r *Renter) ShareFiles(nicknames []string, shareDest string) error {
	lockID := r.mu.RLock()
	defer r.mu.RUnlock(lockID)

	// TODO: consider just appending the proper extension.
	if filepath.Ext(shareDest) != ShareExtension {
		return ErrNonShareSuffix
	}

	handle, err := os.Create(shareDest)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Load files from renter.
	files := make([]*file, len(nicknames))
	for i, name := range nicknames {
		f, exists := r.files[name]
		if !exists {
			return ErrUnknownPath
		}
		files[i] = f
	}

	err = shareFiles(files, handle)
	if err != nil {
		os.Remove(shareDest)
		return err
	}

	return nil
}

// ShareFilesAscii returns the specified files in ASCII format.
func (r *Renter) ShareFilesAscii(nicknames []string) (string, error) {
	lockID := r.mu.RLock()
	defer r.mu.RUnlock(lockID)

	// Load files from renter.
	files := make([]*file, len(nicknames))
	for i, name := range nicknames {
		f, exists := r.files[name]
		if !exists {
			return "", ErrUnknownPath
		}
		files[i] = f
	}

	buf := new(bytes.Buffer)
	err := shareFiles(files, base64.NewEncoder(base64.URLEncoding, buf))
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}

// loadSharedFiles reads .sia data from reader and registers the contained
// files in the renter. It returns the nicknames of the loaded files.
func (r *Renter) loadSharedFiles(reader io.Reader) ([]string, error) {
	// read header
	var header [15]byte
	var version string
	var numFiles uint64
	err := encoding.NewDecoder(reader).DecodeAll(
		&header,
		&version,
		&numFiles,
	)
	if err != nil {
		return nil, err
	} else if header != shareHeader {
		return nil, ErrBadFile
	} else if version != shareVersion {
		return nil, ErrIncompatible
	}

	// Create decompressor.
	unzip, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}
	dec := encoding.NewDecoder(unzip)

	// Read each file.
	files := make([]*file, numFiles)
	for i := range files {
		files[i] = new(file)
		err := dec.Decode(files[i])
		if err != nil {
			return nil, err
		}

		// Make sure the file's name does not conflict with existing files.
		dupCount := 0
		origName := files[i].name
		for {
			_, exists := r.files[files[i].name]
			if !exists {
				break
			}
			dupCount++
			files[i].name = origName + "_" + strconv.Itoa(dupCount)
		}
	}

	// Add files to renter.
	names := make([]string, numFiles)
	for i, f := range files {
		r.files[f.name] = f
		names[i] = f.name
	}
	// Save the files.
	for _, f := range files {
		r.saveFile(f)
	}

	return names, nil
}

// initPersist handles all of the persistence initialization, such as creating
// the persistence directory and starting the logger.
func (r *Renter) initPersist() error {
	// Create the perist directory if it does not yet exist.
	err := os.MkdirAll(r.persistDir, 0700)
	if err != nil {
		return err
	}

	// Initialize the logger.
	r.log, err = persist.NewFileLogger(filepath.Join(r.persistDir, logFile))
	if err != nil {
		return err
	}

	// Load the prior persistence structures.
	err = r.load()
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

// LoadSharedFiles loads a .sia file into the renter. It returns the nicknames
// of the loaded files.
func (r *Renter) LoadSharedFiles(filename string) ([]string, error) {
	lockID := r.mu.Lock()
	defer r.mu.Unlock(lockID)

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return r.loadSharedFiles(file)
}

// LoadSharedFilesAscii loads an ASCII-encoded .sia file into the renter. It
// returns the nicknames of the loaded files.
func (r *Renter) LoadSharedFilesAscii(asciiSia string) ([]string, error) {
	lockID := r.mu.Lock()
	defer r.mu.Unlock(lockID)

	dec := base64.NewDecoder(base64.URLEncoding, bytes.NewBufferString(asciiSia))
	return r.loadSharedFiles(dec)
}
<commit_msg>never return an error from filepath.Walk<commit_after>package renter

import (
	"bytes"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strconv"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

const (
	PersistFilename = "renter.json"
	ShareExtension  = ".sia"
	logFile         = modules.RenterDir + ".log"
)

var (
	ErrNoNicknames    = errors.New("at least one nickname must be supplied")
	ErrNonShareSuffix = errors.New("suffix of file must be " + ShareExtension)
	ErrBadFile        = errors.New("not a .sia file")
	ErrIncompatible   = errors.New("file is not compatible with current version")

	shareHeader  = [15]byte{'S', 'i', 'a', ' ', 'S', 'h', 'a', 'r', 'e', 'd', ' ', 'F', 'i', 'l', 'e'}
	shareVersion = "0.4"

	saveMetadata = persist.Metadata{
		Header:  "Renter Persistence",
		Version: "0.4",
	}
)

// MarshalSia implements the encoding.SiaMarshaller interface, writing the
// file data to w.
func (f *file) MarshalSia(w io.Writer) error {
	enc := encoding.NewEncoder(w)

	// encode easy fields
	err := enc.EncodeAll(
		f.name,
		f.size,
		f.masterKey,
		f.pieceSize,
		f.mode,
	)
	if err != nil {
		return err
	}
	// COMPATv0.4.3 - encode the bytesUploaded and chunksUploaded fields
	// TODO: the resulting .sia file may confuse old clients.
	err = enc.EncodeAll(f.pieceSize*f.numChunks()*uint64(f.erasureCode.NumPieces()), f.numChunks())
	if err != nil {
		return err
	}

	// encode erasureCode
	switch code := f.erasureCode.(type) {
	case *rsCode:
		err = enc.EncodeAll(
			"Reed-Solomon",
			uint64(code.dataPieces),
			uint64(code.numPieces-code.dataPieces),
		)
		if err != nil {
			return err
		}
	default:
		if build.DEBUG {
			panic("unknown erasure code")
		}
		return errors.New("unknown erasure code")
	}
	// encode contracts
	if err := enc.Encode(uint64(len(f.contracts))); err != nil {
		return err
	}
	for _, c := range f.contracts {
		if err := enc.Encode(c); err != nil {
			return err
		}
	}
	return nil
}

// UnmarshalSia implements the encoding.SiaUnmarshaller interface,
// reconstructing a file from the encoded bytes read from r.
func (f *file) UnmarshalSia(r io.Reader) error {
	dec := encoding.NewDecoder(r)

	// COMPATv0.4.3 - decode bytesUploaded and chunksUploaded into dummy vars.
	var bytesUploaded, chunksUploaded uint64

	// decode easy fields
	err := dec.DecodeAll(
		&f.name,
		&f.size,
		&f.masterKey,
		&f.pieceSize,
		&f.mode,
		&bytesUploaded,
		&chunksUploaded,
	)
	if err != nil {
		return err
	}

	// decode erasure coder
	var codeType string
	if err := dec.Decode(&codeType); err != nil {
		return err
	}
	switch codeType {
	case "Reed-Solomon":
		var nData, nParity uint64
		err = dec.DecodeAll(
			&nData,
			&nParity,
		)
		if err != nil {
			return err
		}
		rsc, err := NewRSCode(int(nData), int(nParity))
		if err != nil {
			return err
		}
		f.erasureCode = rsc
	default:
		return errors.New("unrecognized erasure code type: " + codeType)
	}

	// decode contracts
	var nContracts uint64
	if err := dec.Decode(&nContracts); err != nil {
		return err
	}
	f.contracts = make(map[types.FileContractID]fileContract)
	var contract fileContract
	for i := uint64(0); i < nContracts; i++ {
		if err := dec.Decode(&contract); err != nil {
			return err
		}
		f.contracts[contract.ID] = contract
	}
	return nil
}

// saveFile saves a file to the renter directory.
func (r *Renter) saveFile(f *file) error {
	// Create directory structure specified in nickname.
	fullPath := filepath.Join(r.persistDir, f.name+ShareExtension)
	err := os.MkdirAll(filepath.Dir(fullPath), 0700)
	if err != nil {
		return err
	}

	// Open SafeFile handle.
	handle, err := persist.NewSafeFile(filepath.Join(r.persistDir, f.name+ShareExtension))
	if err != nil {
		return err
	}
	defer handle.Close()

	// Write file data.
	err = shareFiles([]*file{f}, handle)
	if err != nil {
		return err
	}

	// Commit the SafeFile.
	return handle.Commit()
}

// save stores the current renter data to disk.
func (r *Renter) save() error {
	data := struct {
		Tracking map[string]trackedFile
	}{r.tracking}
	return persist.SaveFile(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename))
}

// saveSync stores the current renter data to disk and then syncs to disk.
func (r *Renter) saveSync() error {
	data := struct {
		Tracking map[string]trackedFile
	}{r.tracking}
	return persist.SaveFileSync(saveMetadata, data, filepath.Join(r.persistDir, PersistFilename))
}

// load fetches the saved renter data from disk.
func (r *Renter) load() error {
	// Recursively load all files found in renter directory. Errors
	// encountered during loading are logged, but are not considered fatal.
	err := filepath.Walk(r.persistDir, func(path string, info os.FileInfo, err error) error {
		// This error is non-nil if filepath.Walk couldn't stat a file or
		// folder.
		if err != nil {
			r.log.Println("WARN: could not stat file or folder during walk:", err)
			return nil
		}

		// Skip folders and non-sia files.
		if info.IsDir() || filepath.Ext(path) != ShareExtension {
			return nil
		}

		// Open the file.
		file, err := os.Open(path)
		if err != nil {
			r.log.Println("ERROR: could not open .sia file:", err)
			return nil
		}
		defer file.Close()

		// Load the file contents into the renter.
		_, err = r.loadSharedFiles(file)
		if err != nil {
			r.log.Println("ERROR: could not load .sia file:", err)
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Load contracts, repair set, and entropy.
	data := struct {
		Tracking  map[string]trackedFile
		Repairing map[string]string // COMPATv0.4.8
	}{}
	err = persist.LoadFile(saveMetadata, &data, filepath.Join(r.persistDir, PersistFilename))
	if err != nil {
		return err
	}
	if data.Tracking != nil {
		r.tracking = data.Tracking
	}

	return nil
}

// shareFiles writes the specified files to w. First a header is written,
// followed by the gzipped concatenation of each file.
func shareFiles(files []*file, w io.Writer) error {
	// Write header.
	err := encoding.NewEncoder(w).EncodeAll(
		shareHeader,
		shareVersion,
		uint64(len(files)),
	)
	if err != nil {
		return err
	}

	// Create compressor.
	zip, _ := gzip.NewWriterLevel(w, gzip.BestCompression)
	enc := encoding.NewEncoder(zip)

	// Encode each file.
	for _, f := range files {
		err = enc.Encode(f)
		if err != nil {
			return err
		}
	}

	return zip.Close()
}

// ShareFile saves the specified files to shareDest.
func (r *Renter) ShareFiles(nicknames []string, shareDest string) error {
	lockID := r.mu.RLock()
	defer r.mu.RUnlock(lockID)

	// TODO: consider just appending the proper extension.
	if filepath.Ext(shareDest) != ShareExtension {
		return ErrNonShareSuffix
	}

	handle, err := os.Create(shareDest)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Load files from renter.
	files := make([]*file, len(nicknames))
	for i, name := range nicknames {
		f, exists := r.files[name]
		if !exists {
			return ErrUnknownPath
		}
		files[i] = f
	}

	err = shareFiles(files, handle)
	if err != nil {
		os.Remove(shareDest)
		return err
	}

	return nil
}

// ShareFilesAscii returns the specified files in ASCII format.
func (r *Renter) ShareFilesAscii(nicknames []string) (string, error) {
	lockID := r.mu.RLock()
	defer r.mu.RUnlock(lockID)

	// Load files from renter.
	files := make([]*file, len(nicknames))
	for i, name := range nicknames {
		f, exists := r.files[name]
		if !exists {
			return "", ErrUnknownPath
		}
		files[i] = f
	}

	buf := new(bytes.Buffer)
	err := shareFiles(files, base64.NewEncoder(base64.URLEncoding, buf))
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}

// loadSharedFiles reads .sia data from reader and registers the contained
// files in the renter. It returns the nicknames of the loaded files.
func (r *Renter) loadSharedFiles(reader io.Reader) ([]string, error) {
	// read header
	var header [15]byte
	var version string
	var numFiles uint64
	err := encoding.NewDecoder(reader).DecodeAll(
		&header,
		&version,
		&numFiles,
	)
	if err != nil {
		return nil, err
	} else if header != shareHeader {
		return nil, ErrBadFile
	} else if version != shareVersion {
		return nil, ErrIncompatible
	}

	// Create decompressor.
	unzip, err := gzip.NewReader(reader)
	if err != nil {
		return nil, err
	}
	dec := encoding.NewDecoder(unzip)

	// Read each file.
	files := make([]*file, numFiles)
	for i := range files {
		files[i] = new(file)
		err := dec.Decode(files[i])
		if err != nil {
			return nil, err
		}

		// Make sure the file's name does not conflict with existing files.
		dupCount := 0
		origName := files[i].name
		for {
			_, exists := r.files[files[i].name]
			if !exists {
				break
			}
			dupCount++
			files[i].name = origName + "_" + strconv.Itoa(dupCount)
		}
	}

	// Add files to renter.
	names := make([]string, numFiles)
	for i, f := range files {
		r.files[f.name] = f
		names[i] = f.name
	}
	// Save the files.
	for _, f := range files {
		r.saveFile(f)
	}

	return names, nil
}

// initPersist handles all of the persistence initialization, such as creating
// the persistence directory and starting the logger.
func (r *Renter) initPersist() error {
	// Create the perist directory if it does not yet exist.
	err := os.MkdirAll(r.persistDir, 0700)
	if err != nil {
		return err
	}

	// Initialize the logger.
	r.log, err = persist.NewFileLogger(filepath.Join(r.persistDir, logFile))
	if err != nil {
		return err
	}

	// Load the prior persistence structures.
	err = r.load()
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	return nil
}

// LoadSharedFiles loads a .sia file into the renter. It returns the nicknames
// of the loaded files.
func (r *Renter) LoadSharedFiles(filename string) ([]string, error) {
	lockID := r.mu.Lock()
	defer r.mu.Unlock(lockID)

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return r.loadSharedFiles(file)
}

// LoadSharedFilesAscii loads an ASCII-encoded .sia file into the renter. It
// returns the nicknames of the loaded files.
func (r *Renter) LoadSharedFilesAscii(asciiSia string) ([]string, error) {
	lockID := r.mu.Lock()
	defer r.mu.Unlock(lockID)

	dec := base64.NewDecoder(base64.URLEncoding, bytes.NewBufferString(asciiSia))
	return r.loadSharedFiles(dec)
}
<|endoftext|>


<commit_before>package wallet

import (
	"bytes"
	"crypto/rand"
	"errors"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errAlreadyUnlocked   = errors.New("wallet has already been unlocked")
	errReencrypt         = errors.New("wallet is already encrypted, cannot encrypt again")
	errUnencryptedWallet = errors.New("wallet has not been encrypted yet")

	unlockModifier = types.Specifier{'u', 'n', 'l', 'o', 'c', 'k'}
)

// uidEncryptionKey creates an encryption key that is used to decrypt a
// specific key file.
func uidEncryptionKey(masterKey crypto.TwofishKey, uid UniqueID) crypto.TwofishKey {
	return crypto.TwofishKey(crypto.HashAll(masterKey, uid))
}

// checkMasterKey verifies that the master key is correct.
func (w *Wallet) checkMasterKey(masterKey crypto.TwofishKey) error {
	uk := uidEncryptionKey(masterKey, w.persist.UID)
	verification, err := uk.DecryptBytes(w.persist.EncryptionVerification)
	if err != nil {
		// Most of the time, the failure is an authentication failure.
		return modules.ErrBadEncryptionKey
	}
	expected := make([]byte, encryptionVerificationLen)
	if !bytes.Equal(expected, verification) {
		return modules.ErrBadEncryptionKey
	}
	return nil
}

// initEncryption checks that the provided encryption key is the valid
// encryption key for the wallet. If encryption has not yet been established
// for the wallet, an encryption key is created.
func (w *Wallet) initEncryption(masterKey crypto.TwofishKey) (modules.Seed, error) {
	// Check if the wallet encryption key has already been set.
	if len(w.persist.EncryptionVerification) != 0 {
		return modules.Seed{}, errReencrypt
	}

	// Create a random seed and use it to generate the seed file for the
	// wallet.
	var seed modules.Seed
	_, err := rand.Read(seed[:])
	if err != nil {
		return modules.Seed{}, err
	}

	// If the input key is blank, use the seed to create the master key.
	// Otherwise, use the input key.
	if masterKey == (crypto.TwofishKey{}) {
		masterKey = crypto.TwofishKey(crypto.HashObject(seed))
	}
	err = w.createSeed(masterKey, seed)
	if err != nil {
		return modules.Seed{}, err
	}

	// Establish the encryption verification using the masterKey. After this
	// point, the wallet is encrypted.
	uk := uidEncryptionKey(masterKey, w.persist.UID)
	encryptionBase := make([]byte, encryptionVerificationLen)
	w.persist.EncryptionVerification, err = uk.EncryptBytes(encryptionBase)
	if err != nil {
		return modules.Seed{}, err
	}
	err = w.saveSettings()
	if err != nil {
		return modules.Seed{}, err
	}
	return seed, nil
}

// unlock loads all of the encrypted file structures into wallet memory. Even
// after loading, the structures are kept encrypted, but some data such as
// addresses are decrypted so that the wallet knows what to track.
func (w *Wallet) unlock(masterKey crypto.TwofishKey) error {
	// Wallet should only be unlocked once.
	if w.unlocked {
		return errAlreadyUnlocked
	}

	// Check if the wallet encryption key has already been set.
	if len(w.persist.EncryptionVerification) == 0 {
		return errUnencryptedWallet
	}

	// Initialize the encryption of the wallet.
	err := w.checkMasterKey(masterKey)
	if err != nil {
		return err
	}

	// Load the wallet seed that is used to generate new addresses.
	err = w.initPrimarySeed(masterKey)
	if err != nil {
		return err
	}

	// Load all wallet seeds that are not used to generate new addresses.
	err = w.initAuxiliarySeeds(masterKey)
	if err != nil {
		return err
	}

	// Load all keys that were not generated by a seed.
	err = w.initUnseededKeys(masterKey)
	if err != nil {
		return err
	}
	w.unlocked = true
	return nil
}

// wipeSecrets erases all of the seeds and secret keys in the wallet.
func (w *Wallet) wipeSecrets() {
	// 'for i := range' must be used to prevent copies of secret data from
	// being made.
	for i := range w.keys {
		for j := range w.keys[i].SecretKeys {
			crypto.SecureWipe(w.keys[i].SecretKeys[j][:])
		}
	}
	for i := range w.seeds {
		crypto.SecureWipe(w.seeds[i][:])
	}
	crypto.SecureWipe(w.primarySeed[:])
	w.seeds = w.seeds[:0]
}

// Encrypted returns whether or not the wallet has been encrypted.
func (w *Wallet) Encrypted() bool {
	w.mu.Lock()
	defer w.mu.Unlock()
	if build.DEBUG && w.unlocked && len(w.persist.EncryptionVerification) == 0 {
		panic("wallet is both unlocked and unencrypted")
	}
	return len(w.persist.EncryptionVerification) != 0
}

// Encrypt will encrypt the wallet using the input key. Upon encryption, a
// primary seed will be created for the wallet (no seed exists prior to this
// point). If the key is blank, then the hash of the seed that is generated
// will be used as the key. The wallet will still be locked after encryption.
//
// Encrypt can only be called once throughout the life of the wallet, and will
// return an error on subsequent calls (even after restarting the wallet). To
// reset the wallet, the wallet files must be moved to a different directory or
// deleted.
func (w *Wallet) Encrypt(masterKey crypto.TwofishKey) (modules.Seed, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.initEncryption(masterKey)
}

// Unlocked indicates whether the wallet is locked or unlocked.
func (w *Wallet) Unlocked() bool {
	w.mu.RLock()
	defer w.mu.RUnlock()
	return w.unlocked
}

// Lock will erase all keys from memory and prevent the wallet from spending
// coins until it is unlocked.
func (w *Wallet) Lock() error {
	w.mu.RLock()
	defer w.mu.RUnlock()
	if !w.unlocked {
		return modules.ErrLockedWallet
	}
	w.log.Println("INFO: Locking wallet.")

	// Wipe all of the seeds and secret keys, they will be replaced upon
	// calling 'Unlock' again.
	w.wipeSecrets()
	w.unlocked = false
	return w.saveSettings()
}

// Unlock will decrypt the wallet seed and load all of the addresses into
// memory.
func (w *Wallet) Unlock(masterKey crypto.TwofishKey) error {
	w.log.Println("INFO: Unlocking wallet.")

	// Initialize all of the keys in the wallet under a lock. While holding the
	// lock, also grab the subscriber status.
	w.mu.Lock()
	subscribed := w.subscribed
	err := w.unlock(masterKey)
	w.mu.Unlock()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set if this is the first unlock for the
	// wallet object.
	if !subscribed {
		w.cs.ConsensusSetSubscribe(w)
		w.tpool.TransactionPoolSubscribe(w)
		w.mu.Lock()
		w.subscribed = true
		w.mu.Unlock()
	}
	return nil
}
<commit_msg>change readlock to writelock<commit_after>package wallet

import (
	"bytes"
	"crypto/rand"
	"errors"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

var (
	errAlreadyUnlocked   = errors.New("wallet has already been unlocked")
	errReencrypt         = errors.New("wallet is already encrypted, cannot encrypt again")
	errUnencryptedWallet = errors.New("wallet has not been encrypted yet")

	unlockModifier = types.Specifier{'u', 'n', 'l', 'o', 'c', 'k'}
)

// uidEncryptionKey creates an encryption key that is used to decrypt a
// specific key file.
func uidEncryptionKey(masterKey crypto.TwofishKey, uid UniqueID) crypto.TwofishKey {
	return crypto.TwofishKey(crypto.HashAll(masterKey, uid))
}

// checkMasterKey verifies that the master key is correct.
func (w *Wallet) checkMasterKey(masterKey crypto.TwofishKey) error {
	uk := uidEncryptionKey(masterKey, w.persist.UID)
	verification, err := uk.DecryptBytes(w.persist.EncryptionVerification)
	if err != nil {
		// Most of the time, the failure is an authentication failure.
		return modules.ErrBadEncryptionKey
	}
	expected := make([]byte, encryptionVerificationLen)
	if !bytes.Equal(expected, verification) {
		return modules.ErrBadEncryptionKey
	}
	return nil
}

// initEncryption checks that the provided encryption key is the valid
// encryption key for the wallet. If encryption has not yet been established
// for the wallet, an encryption key is created.
func (w *Wallet) initEncryption(masterKey crypto.TwofishKey) (modules.Seed, error) {
	// Check if the wallet encryption key has already been set.
	if len(w.persist.EncryptionVerification) != 0 {
		return modules.Seed{}, errReencrypt
	}

	// Create a random seed and use it to generate the seed file for the
	// wallet.
	var seed modules.Seed
	_, err := rand.Read(seed[:])
	if err != nil {
		return modules.Seed{}, err
	}

	// If the input key is blank, use the seed to create the master key.
	// Otherwise, use the input key.
	if masterKey == (crypto.TwofishKey{}) {
		masterKey = crypto.TwofishKey(crypto.HashObject(seed))
	}
	err = w.createSeed(masterKey, seed)
	if err != nil {
		return modules.Seed{}, err
	}

	// Establish the encryption verification using the masterKey. After this
	// point, the wallet is encrypted.
	uk := uidEncryptionKey(masterKey, w.persist.UID)
	encryptionBase := make([]byte, encryptionVerificationLen)
	w.persist.EncryptionVerification, err = uk.EncryptBytes(encryptionBase)
	if err != nil {
		return modules.Seed{}, err
	}
	err = w.saveSettings()
	if err != nil {
		return modules.Seed{}, err
	}
	return seed, nil
}

// unlock loads all of the encrypted file structures into wallet memory. Even
// after loading, the structures are kept encrypted, but some data such as
// addresses are decrypted so that the wallet knows what to track.
func (w *Wallet) unlock(masterKey crypto.TwofishKey) error {
	// Wallet should only be unlocked once.
	if w.unlocked {
		return errAlreadyUnlocked
	}

	// Check if the wallet encryption key has already been set.
	if len(w.persist.EncryptionVerification) == 0 {
		return errUnencryptedWallet
	}

	// Initialize the encryption of the wallet.
	err := w.checkMasterKey(masterKey)
	if err != nil {
		return err
	}

	// Load the wallet seed that is used to generate new addresses.
	err = w.initPrimarySeed(masterKey)
	if err != nil {
		return err
	}

	// Load all wallet seeds that are not used to generate new addresses.
	err = w.initAuxiliarySeeds(masterKey)
	if err != nil {
		return err
	}

	// Load all keys that were not generated by a seed.
	err = w.initUnseededKeys(masterKey)
	if err != nil {
		return err
	}
	w.unlocked = true
	return nil
}

// wipeSecrets erases all of the seeds and secret keys in the wallet.
func (w *Wallet) wipeSecrets() {
	// 'for i := range' must be used to prevent copies of secret data from
	// being made.
	for i := range w.keys {
		for j := range w.keys[i].SecretKeys {
			crypto.SecureWipe(w.keys[i].SecretKeys[j][:])
		}
	}
	for i := range w.seeds {
		crypto.SecureWipe(w.seeds[i][:])
	}
	crypto.SecureWipe(w.primarySeed[:])
	w.seeds = w.seeds[:0]
}

// Encrypted returns whether or not the wallet has been encrypted.
func (w *Wallet) Encrypted() bool {
	w.mu.Lock()
	defer w.mu.Unlock()
	if build.DEBUG && w.unlocked && len(w.persist.EncryptionVerification) == 0 {
		panic("wallet is both unlocked and unencrypted")
	}
	return len(w.persist.EncryptionVerification) != 0
}

// Encrypt will encrypt the wallet using the input key. Upon encryption, a
// primary seed will be created for the wallet (no seed exists prior to this
// point). If the key is blank, then the hash of the seed that is generated
// will be used as the key. The wallet will still be locked after encryption.
//
// Encrypt can only be called once throughout the life of the wallet, and will
// return an error on subsequent calls (even after restarting the wallet). To
// reset the wallet, the wallet files must be moved to a different directory or
// deleted.
func (w *Wallet) Encrypt(masterKey crypto.TwofishKey) (modules.Seed, error) {
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.initEncryption(masterKey)
}

// Unlocked indicates whether the wallet is locked or unlocked.
func (w *Wallet) Unlocked() bool {
	w.mu.RLock()
	defer w.mu.RUnlock()
	return w.unlocked
}

// Lock will erase all keys from memory and prevent the wallet from spending
// coins until it is unlocked.
func (w *Wallet) Lock() error {
	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.unlocked {
		return modules.ErrLockedWallet
	}
	w.log.Println("INFO: Locking wallet.")

	// Wipe all of the seeds and secret keys, they will be replaced upon
	// calling 'Unlock' again.
	w.wipeSecrets()
	w.unlocked = false
	return w.saveSettings()
}

// Unlock will decrypt the wallet seed and load all of the addresses into
// memory.
func (w *Wallet) Unlock(masterKey crypto.TwofishKey) error {
	w.log.Println("INFO: Unlocking wallet.")

	// Initialize all of the keys in the wallet under a lock. While holding the
	// lock, also grab the subscriber status.
	w.mu.Lock()
	subscribed := w.subscribed
	err := w.unlock(masterKey)
	w.mu.Unlock()
	if err != nil {
		return err
	}

	// Subscribe to the consensus set if this is the first unlock for the
	// wallet object.
	if !subscribed {
		w.cs.ConsensusSetSubscribe(w)
		w.tpool.TransactionPoolSubscribe(w)
		w.mu.Lock()
		w.subscribed = true
		w.mu.Unlock()
	}
	return nil
}
<|endoftext|>