<commit_before>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

/*
	CommitBranchIndex

	Given a repo and a clock, returns a commit

	Is used in several places to:

	- find a parent commit given the parent's id in the form of an alias (e.g. "master/0")
	- getHeadOfBranch() -- by doing a range query of the form "branchName/0" to "branchName/max" and returning the last result (in this case the head)
	- getIDOfParentcommit() -- by decrementing this commit's clock value, and searching for that new clock
	- getCommitByAmbmiguousID() -- if the commit ID is in the form of an alias, find the commit using the index

*/

func TestCommitBranchIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(CommitBranchIndex.GetName(), key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranchRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		// master exists, when providing parentID and branch, assume its a new branch
		require.YesError(t, err)

		err = d.StartCommit(
			repo,
			commitID2,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(CommitBranchIndex.GetName()),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

/*
	diffPathIndex

	used nowhere?
	used in ListFile() to list diffs by path
*/

/* diffCommitIndex

Indexed on commitID field in diff row

Used to:

- in FinishCommit() to gather all of the diffs for this commit

*/

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(DiffCommitIndex.GetName(), commitID).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))

		block := diff.BlockRefs[0]
		fmt.Printf("block: %v\n", block)
		blockSize := block.Upper - block.Lower

		// Was trying to check on a per block level ...
		// But even GetFile() doesn't seem to return the correct results
		// reader, err := d.GetFile(file, nil, 0,
		//	int64(blockSize), &pfs.Commit{Repo: repo, ID: commitID}, 0, false, "")

		reader, err := client.GetBlock(block.Hash, uint64(0), uint64(blockSize))
		require.NoError(t, err)
		var data []byte
		size, err := reader.Read(data)
		fmt.Printf("data=%v, err=%v\n", string(data), err)
		fmt.Printf("size=%v\n", size)
		require.NoError(t, err)
		require.Equal(t, "foo\n", string(data))

	})
}

/* clockBranchIndex

Indexed on:

- repo && branchIndex

Used to:

- in ListBranch() to query the clocks table and return the branches

*/

func TestDiffPathIndexBasicRF(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "repo1"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "foo/bar/fizz/buzz",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("aaa\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		branchClock := &persist.BranchClock{
			Clocks: []*persist.Clock{
				{
					Branch: "master",
					Clock:  0,
				},
			},
		}
		key := []interface{}{repo.Name, false, "foo/bar/fizz/buzz", branchClock.ToArray()}
		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(DiffPathIndex.GetName(), key).Map(DiffPathIndex.GetCreateFunction()).Run(dbClient)

		cursor, err = gorethink.DB(dbName).Table(diffTable).Map(DiffPathIndex.GetCreateFunction()).Run(dbClient)
		require.NoError(t, err)
		fields := []interface{}{}
		require.NoError(t, cursor.One(&fields))
		// Example return value:
		// []interface {}{"repo1", false, "foo/bar/fizz/buzz", []interface {}{[]interface {}{"master", 0}}}
		innerFields, ok := fields[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, repo.Name, innerFields[0].(string))
		require.Equal(t, false, innerFields[1].(bool))
		require.Equal(t, "foo/bar/fizz/buzz", innerFields[2].(string))
		clocks, ok := innerFields[3].([]interface{})
		require.Equal(t, true, ok)
		clock, ok := clocks[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, "master", clock[0].(string))
		require.Equal(t, float64(0), clock[1].(float64))
	})
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session, pclient.APIClient)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	_client, d := startBlockServerAndGetClient(t, dbName)

	testCode(d, dbName, dbClient, _client)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServerAndGetClient(t *testing.T, dbName string) (pclient.APIClient, drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	var driver drive.Driver
	for i, port := range ports {
		address := addresses[i]
		_driver, err := NewDriver(address, RethinkAddress, dbName)
		driver = _driver
		require.NoError(t, err)
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	clientConn, err := grpc.Dial(addresses[0], grpc.WithInsecure())
	require.NoError(t, err)
	return pclient.APIClient{BlockAPIClient: pfsclient.NewBlockAPIClient(clientConn)}, driver
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<commit_msg>cleanup test comments<commit_after>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

func TestCommitBranchIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(CommitBranchIndex.GetName(), key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranchRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		// master exists, when providing parentID and branch, assume its a new branch
		require.YesError(t, err)

		err = d.StartCommit(
			repo,
			commitID2,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(CommitBranchIndex.GetName()),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

func TestDiffCommitIndexBasic(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(DiffCommitIndex.GetName(), commitID).Run(dbClient)
		require.NoError(t, err)
		diff := &persist.Diff{}
		require.NoError(t, cursor.One(diff))
		fmt.Printf("got first diff: %v\n", diff)
		require.Equal(t, "file", diff.Path)
		require.Equal(t, 1, len(diff.BlockRefs))

		block := diff.BlockRefs[0]
		fmt.Printf("block: %v\n", block)
		blockSize := block.Upper - block.Lower

		// Was trying to check on a per block level ...
		// But even GetFile() doesn't seem to return the correct results
		// reader, err := d.GetFile(file, nil, 0,
		//	int64(blockSize), &pfs.Commit{Repo: repo, ID: commitID}, 0, false, "")

		reader, err := client.GetBlock(block.Hash, uint64(0), uint64(blockSize))
		require.NoError(t, err)
		var data []byte
		size, err := reader.Read(data)
		fmt.Printf("data=%v, err=%v\n", string(data), err)
		fmt.Printf("size=%v\n", size)
		require.NoError(t, err)
		require.Equal(t, "foo\n", string(data))

	})
}

func TestDiffPathIndexBasicRF(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "repo1"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "foo/bar/fizz/buzz",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("aaa\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		branchClock := &persist.BranchClock{
			Clocks: []*persist.Clock{
				{
					Branch: "master",
					Clock:  0,
				},
			},
		}
		key := []interface{}{repo.Name, false, "foo/bar/fizz/buzz", branchClock.ToArray()}
		cursor, err := gorethink.DB(dbName).Table(diffTable).GetAllByIndex(DiffPathIndex.GetName(), key).Map(DiffPathIndex.GetCreateFunction()).Run(dbClient)

		cursor, err = gorethink.DB(dbName).Table(diffTable).Map(DiffPathIndex.GetCreateFunction()).Run(dbClient)
		require.NoError(t, err)
		fields := []interface{}{}
		require.NoError(t, cursor.One(&fields))
		// Example return value:
		// []interface {}{"repo1", false, "foo/bar/fizz/buzz", []interface {}{[]interface {}{"master", 0}}}
		innerFields, ok := fields[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, repo.Name, innerFields[0].(string))
		require.Equal(t, false, innerFields[1].(bool))
		require.Equal(t, "foo/bar/fizz/buzz", innerFields[2].(string))
		clocks, ok := innerFields[3].([]interface{})
		require.Equal(t, true, ok)
		clock, ok := clocks[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, "master", clock[0].(string))
		require.Equal(t, float64(0), clock[1].(float64))
	})
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func testSetup(t *testing.T, testCode func(drive.Driver, string, *gorethink.Session, pclient.APIClient)) {
	dbName := "pachyderm_test_" + uuid.NewWithoutDashes()[0:12]
	if err := InitDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
	dbClient, err := gorethink.Connect(gorethink.ConnectOpts{
		Address: RethinkAddress,
		Timeout: connectTimeoutSeconds * time.Second,
	})
	require.NoError(t, err)
	_client, d := startBlockServerAndGetClient(t, dbName)

	testCode(d, dbName, dbClient, _client)

	if err := RemoveDB(RethinkAddress, dbName); err != nil {
		require.NoError(t, err)
		return
	}
}

func runServers(t *testing.T, port int32, blockAPIServer pfsclient.BlockAPIServer) {
	ready := make(chan bool)
	go func() {
		err := protoserver.Serve(
			func(s *grpc.Server) {
				pfsclient.RegisterBlockAPIServer(s, blockAPIServer)
				close(ready)
			},
			protoserver.ServeOptions{Version: version.Version},
			protoserver.ServeEnv{GRPCPort: uint16(port)},
		)
		require.NoError(t, err)
	}()
	<-ready
}

func startBlockServerAndGetClient(t *testing.T, dbName string) (pclient.APIClient, drive.Driver) {

	root := uniqueString("/tmp/pach_test/run")
	var ports []int32
	ports = append(ports, atomic.AddInt32(&port, 1))
	var addresses []string
	for _, port := range ports {
		addresses = append(addresses, fmt.Sprintf("localhost:%d", port))
	}
	var driver drive.Driver
	for i, port := range ports {
		address := addresses[i]
		_driver, err := NewDriver(address, RethinkAddress, dbName)
		driver = _driver
		require.NoError(t, err)
		blockAPIServer, err := pfsserver.NewLocalBlockAPIServer(root)
		require.NoError(t, err)
		runServers(t, port, blockAPIServer)
	}
	clientConn, err := grpc.Dial(addresses[0], grpc.WithInsecure())
	require.NoError(t, err)
	return pclient.APIClient{BlockAPIClient: pfsclient.NewBlockAPIClient(clientConn)}, driver
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<|endoftext|>


<commit_before>package yandexexport

var CloudInitScript string = `#!/usr/bin/env bash
GetMetadata () {
  echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
}

GetInstanceId () {
  echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)"
}

GetServiceAccountId () {
  yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' '
}

InstallYc () {
  curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local
}

InstallAwsCli () {
  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
  unzip -o awscliv2.zip > /dev/null
  sudo ./aws/install
}

InstallPackages () {
  sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
}

InstallTools () {
  InstallPackages
  InstallYc
  InstallAwsCli
}

IMAGE_ID=$(GetMetadata image_id)
INSTANCE_ID=$(GetInstanceId)
DISKNAME=${INSTANCE_ID}-toexport
PATHS=$(GetMetadata paths)
ZONE=$(GetMetadata zone)

Exit () {
  for i in ${PATHS}; do
    LOGDEST="${i}.exporter.log"
    echo "Uploading exporter log to ${LOGDEST}..."
    aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST}
  done

  echo "Delete static access key..."
  if ! yc iam access-key delete ${YC_SK_ID} ; then
    echo "Failed to delete static access key."
    FAIL=1
  fi

  if [ $1 -ne 0 ]; then
	echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value"
    if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-error ; then
	  echo "Failed to update metadata key 'cloud-init-status'."
	  exit 111
	fi
  fi

  exit $1
}

InstallTools

echo "####### Export configuration #######"
echo "Image ID - ${IMAGE_ID}"
echo "Instance ID - ${INSTANCE_ID}"
echo "Instance zone - ${ZONE}"
echo "Disk name - ${DISKNAME}"
echo "Export paths - ${PATHS}"
echo "####################################"

echo "Detect Service Account ID..."
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"

echo "Create static access key..."
SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \
    --description "this key is for export image to storage" --format json)

if [ $? -ne 0 ]; then
  echo "Failed to create static access key."
  Exit 1
fi

echo "Setup env variables to access storage..."
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json}  )"

for i in ${PATHS}; do
  bucket=$(echo ${i} | sed 's/\(s3:\/\/[^\/]*\).*/\1/')
  echo "Check access to storage: '${bucket}'..."
  if ! aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net ls ${bucket} > /dev/null ; then
    echo "Failed to access storage: '${bucket}'."
    Exit 1
  fi
done

echo "Creating disk from image to be exported..."
if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then
  echo "Failed to create disk."
  Exit 1
fi

echo "Attaching disk..."
if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then
  echo "Failed to attach disk."
  Exit 1
fi

DISK_LINK="/dev/disk/by-id/virtio-doexport"
echo "Waiting for disk..."
for attempt in 1 2 3; do
  sleep 3
  if [ -L "${DISK_LINK}" ]; then
    break
  fi
  echo "Attempt ${attempt}"
  if [ ${attempt} -eq 3 ]; then
    echo "Symlink ${DISK_LINK} not found"
    Exit 1
  fi
done

echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_LINK}" disk.qcow2 ; then
  echo "Failed to dump disk to qcow2 image."
  Exit 1
fi

echo "Detaching disk..."
if ! yc compute instance detach-disk ${INSTANCE_ID}  --disk-name ${DISKNAME} ; then
  echo "Failed to detach disk."
fi

FAIL=0
echo "Deleting disk..."
if ! yc compute disk delete --name ${DISKNAME} ; then
  echo "Failed to delete disk."
  FAIL=1
fi
for i in ${PATHS}; do
  echo "Uploading qcow2 disk image to ${i}..."
  if ! aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then
    echo "Failed to upload image to ${i}."
    FAIL=1
  fi
done

echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value"
if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then
  echo "Failed to update metadata key to 'cloud-init-status'."
  Exit 1
fi

Exit ${FAIL}`
<commit_msg>manually trigger udev<commit_after>package yandexexport

var CloudInitScript string = `#!/usr/bin/env bash
GetMetadata () {
  echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
}

GetInstanceId () {
  echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)"
}

GetServiceAccountId () {
  yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' '
}

InstallYc () {
  curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local
}

InstallAwsCli () {
  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
  unzip -o awscliv2.zip > /dev/null
  sudo ./aws/install
}

InstallPackages () {
  sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils
}

InstallTools () {
  InstallPackages
  InstallYc
  InstallAwsCli
}

IMAGE_ID=$(GetMetadata image_id)
INSTANCE_ID=$(GetInstanceId)
DISKNAME=${INSTANCE_ID}-toexport
PATHS=$(GetMetadata paths)
ZONE=$(GetMetadata zone)

Exit () {
  for i in ${PATHS}; do
    LOGDEST="${i}.exporter.log"
    echo "Uploading exporter log to ${LOGDEST}..."
    aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST}
  done

  echo "Delete static access key..."
  if ! yc iam access-key delete ${YC_SK_ID} ; then
    echo "Failed to delete static access key."
    FAIL=1
  fi

  if [ $1 -ne 0 ]; then
	echo "Set metadata key 'cloud-init-status' to 'cloud-init-error' value"
    if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-error ; then
	  echo "Failed to update metadata key 'cloud-init-status'."
	  exit 111
	fi
  fi

  exit $1
}

InstallTools

echo "####### Export configuration #######"
echo "Image ID - ${IMAGE_ID}"
echo "Instance ID - ${INSTANCE_ID}"
echo "Instance zone - ${ZONE}"
echo "Disk name - ${DISKNAME}"
echo "Export paths - ${PATHS}"
echo "####################################"

echo "Detect Service Account ID..."
SERVICE_ACCOUNT_ID=$(GetServiceAccountId)
echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}"

echo "Create static access key..."
SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \
    --description "this key is for export image to storage" --format json)

if [ $? -ne 0 ]; then
  echo "Failed to create static access key."
  Exit 1
fi

echo "Setup env variables to access storage..."
eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json}  )"

for i in ${PATHS}; do
  bucket=$(echo ${i} | sed 's/\(s3:\/\/[^\/]*\).*/\1/')
  echo "Check access to storage: '${bucket}'..."
  if ! aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net ls ${bucket} > /dev/null ; then
    echo "Failed to access storage: '${bucket}'."
    Exit 1
  fi
done

echo "Creating disk from image to be exported..."
if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then
  echo "Failed to create disk."
  Exit 1
fi

echo "Attaching disk..."
if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then
  echo "Failed to attach disk."
  Exit 1
fi

DISK_LINK="/dev/disk/by-id/virtio-doexport"
echo "Waiting for disk..."
for attempt in 1 2 3; do
  sleep 3
  /sbin/udevadm trigger
  if [ -L "${DISK_LINK}" ]; then
    break
  fi
  echo "Attempt ${attempt}"
  if [ ${attempt} -eq 3 ]; then
    echo "Symlink ${DISK_LINK} not found"
    Exit 1
  fi
done

echo "Dumping disk..."
if ! qemu-img convert -O qcow2 -o cluster_size=2M "${DISK_LINK}" disk.qcow2 ; then
  echo "Failed to dump disk to qcow2 image."
  Exit 1
fi

echo "Detaching disk..."
if ! yc compute instance detach-disk ${INSTANCE_ID}  --disk-name ${DISKNAME} ; then
  echo "Failed to detach disk."
fi

FAIL=0
echo "Deleting disk..."
if ! yc compute disk delete --name ${DISKNAME} ; then
  echo "Failed to delete disk."
  FAIL=1
fi
for i in ${PATHS}; do
  echo "Uploading qcow2 disk image to ${i}..."
  if ! aws s3 --region ru-central1 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then
    echo "Failed to upload image to ${i}."
    FAIL=1
  fi
done

echo "Set metadata key 'cloud-init-status' to 'cloud-init-done' value"
if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then
  echo "Failed to update metadata key to 'cloud-init-status'."
  Exit 1
fi

Exit ${FAIL}`
<|endoftext|>


<commit_before>package main

import (
	"time"
	"fmt"
	"os"
	"strings"
	"strconv"
	"os/exec"
	"log"
	"io/ioutil"
)

type TimelapseState struct {
	headFromState time.Time

	ledSupported bool

	// stills

	stillMaxNumber int
	dirStills string
	contentsStills []string
	dirStillsTemp string

	// 5 min

	dir5min string
	contents5mins []string

	// hourly

	dirHour string
	contentsHours []string

	// daily (no need to track contents as we don't merge them)

	dirDaily string
}

/*
	LED support
	-----------

	Set LED to trigger on GPIO:

	$ sudo su

	Install WiringPi

	$ git clone git://git.drogon.net/wiringPi && cd wiringPi && ./build

	Test LED

	$ gpio -g mode 16 output
	$ gpio -g write 16 1 # off
	$ gpio -g write 16 0 # on

	To have avimerge
	----------------

	$ apt-get install -y transcode
	$ avimerge -v
	avimerge (transcode v1.1.7) (C) 2001-2004 Thomas Oestreich, T. Bitterberg 2004-2010 Transcode Team

*/
// Install
//
// $ sudo apt-get install libav-tools
//
// https://www.raspberrypi.org/forums/viewtopic.php?t=72435
// 	$ sudo sh -c 'echo deb http://vontaene.de/raspbian-updates/ . main >> /etc/apt/sources.list'
// 	$ sudo apt-get install libgstreamer1.0-0 liborc-0.4-0 gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 gstreamer1.0-alsa gstreamer1.0-omx gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio gstreamer1.0-tools gstreamer1.0-x libgstreamer-plugins-bad1.0-0 libgstreamer-plugins-base1.0-0

/*	This assumes that you've done the following:

	$ echo gpio >/sys/class/leds/led0/trigger
	$ gpio -g mode 16 output
*/
func setRaspberryPowerLed(on bool, state *TimelapseState) {
	if !state.ledSupported {
		return
	}

	bitStatus := "0" // for on (SIC)

	if !on {
		bitStatus = "1" // for off
	}

	exec.Command("gpio", "-g", "write", "16", bitStatus).Start()

	// log.Printf("setRaspberryPowerLed: bit = %s (0 => on, 1 => off)", bitStatus)
}

func timeToLast5Mins(ts time.Time) time.Time {
	/*
		10:00:00 -> 10:00
		10:04:59 -> 10:00
		10:05:01 -> 10:05
	*/

	min := ts.Minute()

	return time.Date(ts.Year(), ts.Month(), ts.Day(), ts.Hour(), min - (min % 5), 0, 0, time.UTC)
}

/*
func tickEverySecond(tickChan chan int) {
	for {
		now := time.Now()
		nextSec := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second() + 1, 0, time.UTC)

		time.Sleep(nextSec.Sub(now))

		// tickChan <- nextSec.String()
		tickChan <- TICK_1SEC
	}
}
*/

func makeDirIfNotExists(dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		log.Printf("makeDirIfNotExists: making %s", dir)

		if err = os.Mkdir(dir, 0755); err != nil {
			panic(err)
		}
	}
}

func restoreState() TimelapseState {
	state := TimelapseState{}
	state.stillMaxNumber = 0
	state.headFromState = time.Time{} // TODO: done automatically?
	state.dirStills = "/home/pi/timelapse/bucket_stills"
	state.dirStillsTemp = "/home/pi/timelapse/bucket_stills_temp"
	state.dir5min = "/home/pi/timelapse/bucket_5min"
	state.dirHour = "/home/pi/timelapse/bucket_hour"
	state.dirDaily = "/home/pi/timelapse/bucket_day"

	// dirStillsTemp left out on purpose because dirStills is renamed to it
	makeDirIfNotExists(state.dirStills)
	makeDirIfNotExists(state.dir5min)
	makeDirIfNotExists(state.dirHour)
	makeDirIfNotExists(state.dirDaily)

	// TODO: detect LED support
	state.ledSupported = true

	// ----------- read stills

	files, err := ioutil.ReadDir(state.dirStills)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		if state.headFromState.IsZero() {
			state.headFromState = timeToLast5Mins(file.ModTime())
		}
		state.contentsStills = append(state.contentsStills, state.dirStills + "/" + file.Name())

		state.stillMaxNumber++
	}

	// ----------- read 5mins

	files, err = ioutil.ReadDir(state.dir5min)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		state.contents5mins = append(state.contents5mins, state.dir5min + "/" + file.Name())
	}

	// ----------- read hours

	files, err = ioutil.ReadDir(state.dirHour)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		state.contentsHours = append(state.contentsHours, state.dirHour + "/" + file.Name())
	}

	// ----------- process the rest

	if state.headFromState.IsZero() {
		state.headFromState = timeToLast5Mins(time.Now())
	}

	return state
}

/*
$ tree timelapse/
timelapse/
___ bucket_5min
___ bucket_day
___ bucket_hour
___ bucket_minute
___ ___ 1.jpg
___ ___ 2.jpg
___ ___ 3.jpg

*/

func stillsTo5minBootstrap(state *TimelapseState) {
	log.Printf("stillsTo5minBootstrap: %s -> %s", state.dirStills, state.dirStillsTemp)

	// /home/pi/timelapse/bucket_stills => /home/pi/timelapse/bucket_stills_temp
	err := os.Rename(state.dirStills, state.dirStillsTemp)
	if err != nil {
		panic(err)
	}

	err = os.Mkdir(state.dirStills, 0755)
	if err != nil {
		panic(err)
	}

	// reset still counter
	state.stillMaxNumber = 0

	// clear all stills
	state.contentsStills = []string{}
}

func stillsTo5min(state *TimelapseState) {
	fiveMinVideoFilename := state.headFromState.Format("2006-01-02_03-04.avi")

	outFile := state.dir5min + "/" + fiveMinVideoFilename

	state.contents5mins = append(state.contents5mins, outFile)

	// gst-launch-1.0 multifilesrc location=%d.jpg index=1 caps="image/jpeg,framerate=24/1" ! jpegdec ! omxh264enc ! avimux ! filesink location=timelapse.avi
	args := []string{
		"gst-launch-1.0",
		"multifilesrc",
		"location=" + state.dirStillsTemp + "/%d.jpg",
		"index=1",
		"caps=image/jpeg,framerate=24/1",
		"!", "jpegdec",
		"!", "omxh264enc",
		// "!", "omxh264enc", "target-bitrate=800000000",
		"!", "avimux", // use mp4mux?
		"!", "filesink", "location=" + outFile,
	}

	// avconv -f image2 -i /home/pi/timelapse/bucket_stills_temp/%d.jpg -r 12 -s 1920x1440 /home/pi/timelapse/bucket_5min/foo.mkv
	// args := []string{"avconv", "-f", "image2", "-i", state.dirStillsTemp + "/%d.jpg", "-r", "12", "-s", "1920x1440", outFile}

	log.Printf("stillsTo5min: invoking %s\n", strings.Join(args, " "))

	argv := args[1:]
	cmd := exec.Command(args[0], argv...)
	if output, err2 := cmd.CombinedOutput(); err2 != nil {
		fmt.Fprintln(os.Stderr, string(output), err2)
		panic(err2)
	}

	log.Printf("stillsTo5min: done, deleting %s\n", state.dirStillsTemp)

	if err3 := os.RemoveAll(state.dirStillsTemp); err3 != nil {
		panic(err3)
	}
}

func mergeVideosInternal(files []string, outFile string) {
	// avimerge -o merged.avi -i in1.avi in2.avi
	args := []string{
		"avimerge",
		"-o", outFile,
	}

	for _, file := range files {
		args = append(args, "-i", file)
	}

	log.Printf("mergeVideosInternal: invoking %s\n", strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	if output, err := cmd.CombinedOutput(); err != nil {
		fmt.Fprintln(os.Stderr, string(output), err)
		panic(err)
	}

	log.Printf("mergeVideosInternal: done. deleting %d file(s)\n", len(files))

	for _, fileToDelete := range files {
		if deleteErr := os.Remove(fileToDelete); deleteErr != nil {
			panic(deleteErr)
		}
	}
}

func fiveMinsToHour(state *TimelapseState) {
	hourVideoMerged := state.dirHour + "/" + state.headFromState.Format("2006-01-02_03.avi")

	log.Printf("fiveMinsToHour: %s -> %s", state.dir5min, hourVideoMerged)

	// TODO: make goroutines
	mergeVideosInternal(state.contents5mins, hourVideoMerged)

	state.contents5mins = []string{}
	state.contentsHours = append(state.contentsHours, hourVideoMerged)
}

func hoursToDay(state *TimelapseState) {
	dayVideoMerged := state.dirDaily + "/" + state.headFromState.Format("2006-01-02.avi")

	log.Printf("hoursToDay: %s -> %s", state.dirHour, dayVideoMerged)

	// TODO: make goroutines
	mergeVideosInternal(state.contentsHours, dayVideoMerged)

	state.contentsHours = []string{}
}

func takeStill(state *TimelapseState) {
	stillNumber := state.stillMaxNumber

	state.stillMaxNumber = state.stillMaxNumber + 1

	args := []string{
		"raspistill",
		"-t", "1000",
		"-w", "1280",
		"-h", "960",
		"-o", state.dirStills + "/" + strconv.Itoa(stillNumber) + ".jpg",
	}

	// log.Printf("takeStill: invoking %s", strings.Join(args, " "))
	log.Printf("takeStill: starting")

	argv := args[1:]
	cmd := exec.Command(args[0], argv...)

	setRaspberryPowerLed(true, state)

	if output, err := cmd.CombinedOutput(); err != nil {
		fmt.Fprintln(os.Stderr, string(output), err)
		panic(err)
	}

	setRaspberryPowerLed(false, state)
}

func main() {
	state := restoreState()

	setRaspberryPowerLed(false, &state)

	for {
		now := time.Now()
		// in 5 seconds (TODO: this will drift a bit)
		nextTickShouldBe := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second() + 5, 0, time.UTC)
		nowAccordingToInternet := timeToLast5Mins(now)

		differentDay := nowAccordingToInternet.Day() != state.headFromState.Day()
		differentHour := differentDay || nowAccordingToInternet.Hour() != state.headFromState.Hour()
		different5Min := !nowAccordingToInternet.Equal(state.headFromState)

		// TODO: have these run concurrently WRT takeStil(), so these periodical chores do not delay takeStill()
		if different5Min {
			stillsTo5minBootstrap(&state)

			runRestConcurrently := func () {
				stillsTo5min(&state)

				// different5Min is always true along with differentHour || differentDay

				if differentHour {
					// depends on result of stillsTo5min()
					fiveMinsToHour(&state)
				}

				if differentDay {
					// depends on result of fiveMinsToHour()
					hoursToDay(&state)
				}
			}

			go runRestConcurrently()
		}

		// should run right after stillsTo5min() bootstrap (dir rename, new dir) is done
		takeStill(&state)

		state.headFromState = nowAccordingToInternet

		durationToNextTick := nextTickShouldBe.Sub(time.Now())

		if (durationToNextTick > 0) { // can be negative (if we're late) or 0
			time.Sleep(durationToNextTick)
		}
	}
}
<commit_msg>It's not a power LED<commit_after>package main

import (
	"time"
	"fmt"
	"os"
	"strings"
	"strconv"
	"os/exec"
	"log"
	"io/ioutil"
)

type TimelapseState struct {
	headFromState time.Time

	ledSupported bool

	// stills

	stillMaxNumber int
	dirStills string
	contentsStills []string
	dirStillsTemp string

	// 5 min

	dir5min string
	contents5mins []string

	// hourly

	dirHour string
	contentsHours []string

	// daily (no need to track contents as we don't merge them)

	dirDaily string
}

/*
	LED support
	-----------

	Set LED to trigger on GPIO:

	$ sudo su

	Install WiringPi

	$ git clone git://git.drogon.net/wiringPi && cd wiringPi && ./build

	Test LED

	$ gpio -g mode 16 output
	$ gpio -g write 16 1 # off
	$ gpio -g write 16 0 # on

	To have avimerge
	----------------

	$ apt-get install -y transcode
	$ avimerge -v
	avimerge (transcode v1.1.7) (C) 2001-2004 Thomas Oestreich, T. Bitterberg 2004-2010 Transcode Team

*/
// Install
//
// $ sudo apt-get install libav-tools
//
// https://www.raspberrypi.org/forums/viewtopic.php?t=72435
// 	$ sudo sh -c 'echo deb http://vontaene.de/raspbian-updates/ . main >> /etc/apt/sources.list'
// 	$ sudo apt-get install libgstreamer1.0-0 liborc-0.4-0 gir1.2-gst-plugins-base-1.0 gir1.2-gstreamer-1.0 gstreamer1.0-alsa gstreamer1.0-omx gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-base-apps gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly gstreamer1.0-pulseaudio gstreamer1.0-tools gstreamer1.0-x libgstreamer-plugins-bad1.0-0 libgstreamer-plugins-base1.0-0

/*	This assumes that you've done the following:

	$ echo gpio >/sys/class/leds/led0/trigger
	$ gpio -g mode 16 output
*/
func setRaspberryLed(on bool, state *TimelapseState) {
	if !state.ledSupported {
		return
	}

	bitStatus := "0" // for on (SIC)

	if !on {
		bitStatus = "1" // for off
	}

	exec.Command("gpio", "-g", "write", "16", bitStatus).Start()

	// log.Printf("setRaspberryLed: bit = %s (0 => on, 1 => off)", bitStatus)
}

func timeToLast5Mins(ts time.Time) time.Time {
	/*
		10:00:00 -> 10:00
		10:04:59 -> 10:00
		10:05:01 -> 10:05
	*/

	min := ts.Minute()

	return time.Date(ts.Year(), ts.Month(), ts.Day(), ts.Hour(), min - (min % 5), 0, 0, time.UTC)
}

/*
func tickEverySecond(tickChan chan int) {
	for {
		now := time.Now()
		nextSec := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second() + 1, 0, time.UTC)

		time.Sleep(nextSec.Sub(now))

		// tickChan <- nextSec.String()
		tickChan <- TICK_1SEC
	}
}
*/

func makeDirIfNotExists(dir string) {
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		log.Printf("makeDirIfNotExists: making %s", dir)

		if err = os.Mkdir(dir, 0755); err != nil {
			panic(err)
		}
	}
}

func restoreState() TimelapseState {
	state := TimelapseState{}
	state.stillMaxNumber = 0
	state.headFromState = time.Time{} // TODO: done automatically?
	state.dirStills = "/home/pi/timelapse/bucket_stills"
	state.dirStillsTemp = "/home/pi/timelapse/bucket_stills_temp"
	state.dir5min = "/home/pi/timelapse/bucket_5min"
	state.dirHour = "/home/pi/timelapse/bucket_hour"
	state.dirDaily = "/home/pi/timelapse/bucket_day"

	// dirStillsTemp left out on purpose because dirStills is renamed to it
	makeDirIfNotExists(state.dirStills)
	makeDirIfNotExists(state.dir5min)
	makeDirIfNotExists(state.dirHour)
	makeDirIfNotExists(state.dirDaily)

	// TODO: detect LED support
	state.ledSupported = true

	// ----------- read stills

	files, err := ioutil.ReadDir(state.dirStills)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		if state.headFromState.IsZero() {
			state.headFromState = timeToLast5Mins(file.ModTime())
		}
		state.contentsStills = append(state.contentsStills, state.dirStills + "/" + file.Name())

		state.stillMaxNumber++
	}

	// ----------- read 5mins

	files, err = ioutil.ReadDir(state.dir5min)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		state.contents5mins = append(state.contents5mins, state.dir5min + "/" + file.Name())
	}

	// ----------- read hours

	files, err = ioutil.ReadDir(state.dirHour)
	if err != nil {
		panic(err)
	}

	for _, file := range files {
		state.contentsHours = append(state.contentsHours, state.dirHour + "/" + file.Name())
	}

	// ----------- process the rest

	if state.headFromState.IsZero() {
		state.headFromState = timeToLast5Mins(time.Now())
	}

	return state
}

/*
$ tree timelapse/
timelapse/
___ bucket_5min
___ bucket_day
___ bucket_hour
___ bucket_minute
___ ___ 1.jpg
___ ___ 2.jpg
___ ___ 3.jpg

*/

func stillsTo5minBootstrap(state *TimelapseState) {
	log.Printf("stillsTo5minBootstrap: %s -> %s", state.dirStills, state.dirStillsTemp)

	// /home/pi/timelapse/bucket_stills => /home/pi/timelapse/bucket_stills_temp
	err := os.Rename(state.dirStills, state.dirStillsTemp)
	if err != nil {
		panic(err)
	}

	err = os.Mkdir(state.dirStills, 0755)
	if err != nil {
		panic(err)
	}

	// reset still counter
	state.stillMaxNumber = 0

	// clear all stills
	state.contentsStills = []string{}
}

func stillsTo5min(state *TimelapseState) {
	fiveMinVideoFilename := state.headFromState.Format("2006-01-02_03-04.avi")

	outFile := state.dir5min + "/" + fiveMinVideoFilename

	state.contents5mins = append(state.contents5mins, outFile)

	// gst-launch-1.0 multifilesrc location=%d.jpg index=1 caps="image/jpeg,framerate=24/1" ! jpegdec ! omxh264enc ! avimux ! filesink location=timelapse.avi
	args := []string{
		"gst-launch-1.0",
		"multifilesrc",
		"location=" + state.dirStillsTemp + "/%d.jpg",
		"index=1",
		"caps=image/jpeg,framerate=24/1",
		"!", "jpegdec",
		"!", "omxh264enc",
		// "!", "omxh264enc", "target-bitrate=800000000",
		"!", "avimux", // use mp4mux?
		"!", "filesink", "location=" + outFile,
	}

	// avconv -f image2 -i /home/pi/timelapse/bucket_stills_temp/%d.jpg -r 12 -s 1920x1440 /home/pi/timelapse/bucket_5min/foo.mkv
	// args := []string{"avconv", "-f", "image2", "-i", state.dirStillsTemp + "/%d.jpg", "-r", "12", "-s", "1920x1440", outFile}

	log.Printf("stillsTo5min: invoking %s\n", strings.Join(args, " "))

	argv := args[1:]
	cmd := exec.Command(args[0], argv...)
	if output, err2 := cmd.CombinedOutput(); err2 != nil {
		fmt.Fprintln(os.Stderr, string(output), err2)
		panic(err2)
	}

	log.Printf("stillsTo5min: done, deleting %s\n", state.dirStillsTemp)

	if err3 := os.RemoveAll(state.dirStillsTemp); err3 != nil {
		panic(err3)
	}
}

func mergeVideosInternal(files []string, outFile string) {
	// avimerge -o merged.avi -i in1.avi in2.avi
	args := []string{
		"avimerge",
		"-o", outFile,
	}

	for _, file := range files {
		args = append(args, "-i", file)
	}

	log.Printf("mergeVideosInternal: invoking %s\n", strings.Join(args, " "))

	cmd := exec.Command(args[0], args[1:]...)
	if output, err := cmd.CombinedOutput(); err != nil {
		fmt.Fprintln(os.Stderr, string(output), err)
		panic(err)
	}

	log.Printf("mergeVideosInternal: done. deleting %d file(s)\n", len(files))

	for _, fileToDelete := range files {
		if deleteErr := os.Remove(fileToDelete); deleteErr != nil {
			panic(deleteErr)
		}
	}
}

func fiveMinsToHour(state *TimelapseState) {
	hourVideoMerged := state.dirHour + "/" + state.headFromState.Format("2006-01-02_03.avi")

	log.Printf("fiveMinsToHour: %s -> %s", state.dir5min, hourVideoMerged)

	// TODO: make goroutines
	mergeVideosInternal(state.contents5mins, hourVideoMerged)

	state.contents5mins = []string{}
	state.contentsHours = append(state.contentsHours, hourVideoMerged)
}

func hoursToDay(state *TimelapseState) {
	dayVideoMerged := state.dirDaily + "/" + state.headFromState.Format("2006-01-02.avi")

	log.Printf("hoursToDay: %s -> %s", state.dirHour, dayVideoMerged)

	// TODO: make goroutines
	mergeVideosInternal(state.contentsHours, dayVideoMerged)

	state.contentsHours = []string{}
}

func takeStill(state *TimelapseState) {
	stillNumber := state.stillMaxNumber

	state.stillMaxNumber = state.stillMaxNumber + 1

	args := []string{
		"raspistill",
		"-t", "1000",
		"-w", "1280",
		"-h", "960",
		"-o", state.dirStills + "/" + strconv.Itoa(stillNumber) + ".jpg",
	}

	// log.Printf("takeStill: invoking %s", strings.Join(args, " "))
	log.Printf("takeStill: starting")

	argv := args[1:]
	cmd := exec.Command(args[0], argv...)

	setRaspberryLed(true, state)

	if output, err := cmd.CombinedOutput(); err != nil {
		fmt.Fprintln(os.Stderr, string(output), err)
		panic(err)
	}

	setRaspberryLed(false, state)
}

func main() {
	state := restoreState()

	setRaspberryLed(false, &state)

	for {
		now := time.Now()
		// in 5 seconds (TODO: this will drift a bit)
		nextTickShouldBe := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second() + 5, 0, time.UTC)
		nowAccordingToInternet := timeToLast5Mins(now)

		differentDay := nowAccordingToInternet.Day() != state.headFromState.Day()
		differentHour := differentDay || nowAccordingToInternet.Hour() != state.headFromState.Hour()
		different5Min := !nowAccordingToInternet.Equal(state.headFromState)

		// TODO: have these run concurrently WRT takeStil(), so these periodical chores do not delay takeStill()
		if different5Min {
			stillsTo5minBootstrap(&state)

			runRestConcurrently := func () {
				stillsTo5min(&state)

				// different5Min is always true along with differentHour || differentDay

				if differentHour {
					// depends on result of stillsTo5min()
					fiveMinsToHour(&state)
				}

				if differentDay {
					// depends on result of fiveMinsToHour()
					hoursToDay(&state)
				}
			}

			go runRestConcurrently()
		}

		// should run right after stillsTo5min() bootstrap (dir rename, new dir) is done
		takeStill(&state)

		state.headFromState = nowAccordingToInternet

		durationToNextTick := nextTickShouldBe.Sub(time.Now())

		if (durationToNextTick > 0) { // can be negative (if we're late) or 0
			time.Sleep(durationToNextTick)
		}
	}
}
<|endoftext|>


<commit_before>package models

import (
    "fmt"
    "ds3/networking"
)

type BadStatusCodeError struct {
    ExpectedStatusCode []int
    ActualStatusCode int
    ErrorBody *Error
}

func buildBadStatusCodeError(webResponse networking.WebResponse, expectedStatusCodes []int) *BadStatusCodeError {
    var errorBody Error
    var errorBodyPtr *Error

    // Parse the body and if it worked then use the structure.
    err := parseResponseBody(webResponse.Body(), &errorBody)
    if err == nil {
        errorBodyPtr = &errorBody
    }

    // Return the bad status code entity.
    return &BadStatusCodeError{
        expectedStatusCodes,
        webResponse.StatusCode(),
        errorBodyPtr,
    }
}

func (err BadStatusCodeError) Error() string {
    if err.ErrorBody != nil {
        return fmt.Sprintf(
            "Received a status code of %d when %v was expected. Error message: \"%s\"",
            err.ActualStatusCode,
            err.ExpectedStatusCode,
            err.ErrorBody.Message,
        )
    } else {
        return fmt.Sprintf(
            "Received a status code of %d when %v was expected. Could not parse the response for additional information.",
            err.ActualStatusCode,
            err.ExpectedStatusCode,
        )
    }
}
<commit_msg>Fixed printing of error message<commit_after>package models

import (
    "fmt"
    "ds3/networking"
)

type BadStatusCodeError struct {
    ExpectedStatusCode []int
    ActualStatusCode int
    ErrorBody *Error
}

func buildBadStatusCodeError(webResponse networking.WebResponse, expectedStatusCodes []int) *BadStatusCodeError {
    var errorBody Error
    var errorBodyPtr *Error

    // Parse the body and if it worked then use the structure.
    err := parseResponseBody(webResponse.Body(), &errorBody)
    if err == nil {
        errorBodyPtr = &errorBody
    }

    // Return the bad status code entity.
    return &BadStatusCodeError{
        expectedStatusCodes,
        webResponse.StatusCode(),
        errorBodyPtr,
    }
}

func (err BadStatusCodeError) Error() string {
    if err.ErrorBody != nil && err.ErrorBody.Message != nil {
        return fmt.Sprintf(
            "Received a status code of %d when %v was expected. Error message: \"%s\"",
            err.ActualStatusCode,
            err.ExpectedStatusCode,
            *err.ErrorBody.Message,
        )
    } else {
        return fmt.Sprintf(
            "Received a status code of %d when %v was expected. Could not parse the response for additional information.",
            err.ActualStatusCode,
            err.ExpectedStatusCode,
        )
    }
}
<|endoftext|>


<commit_before>package dnsimple

import (
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers"
	"github.com/StackExchange/dnscontrol/providers/diff"
	"github.com/pkg/errors"

	dnsimpleapi "github.com/dnsimple/dnsimple-go/dnsimple"
)

var features = providers.DocumentationNotes{
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.CanUseSRV:              providers.Can(),
	providers.CanUseTLSA:             providers.Cannot(),
	providers.DocCreateDomains:       providers.Cannot(),
	providers.DocDualHost:            providers.Cannot("DNSimple does not allow sufficient control over the apex NS records"),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	providers.RegisterRegistrarType("DNSIMPLE", newReg)
	providers.RegisterDomainServiceProviderType("DNSIMPLE", newDsp, features)
}

const stateRegistered = "registered"

var defaultNameServerNames = []string{
	"ns1.dnsimple.com",
	"ns2.dnsimple.com",
	"ns3.dnsimple.com",
	"ns4.dnsimple.com",
}

// DnsimpleApi is the handle for this provider.
type DnsimpleApi struct {
	AccountToken string // The account access token
	BaseURL      string // An alternate base URI
	accountID    string // Account id cache
}

// GetNameservers returns the name servers for a domain.
func (c *DnsimpleApi) GetNameservers(domainName string) ([]*models.Nameserver, error) {
	return models.StringsToNameservers(defaultNameServerNames), nil
}

// GetDomainCorrections returns corrections that update a domain.
func (c *DnsimpleApi) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}
	dc.Punycode()
	records, err := c.getRecords(dc.Name)
	if err != nil {
		return nil, err
	}

	var actual []*models.RecordConfig
	for _, r := range records {
		if r.Type == "SOA" || r.Type == "NS" {
			continue
		}
		if r.Name == "" {
			r.Name = "@"
		}
		if r.Type == "CNAME" || r.Type == "MX" || r.Type == "ALIAS" || r.Type == "SRV" {
			r.Content += "."
		}
		// dnsimple adds these odd txt records that mirror the alias records.
		// they seem to manage them on deletes and things, so we'll just pretend they don't exist
		if r.Type == "TXT" && strings.HasPrefix(r.Content, "ALIAS for ") {
			continue
		}
		rec := &models.RecordConfig{
			TTL:      uint32(r.TTL),
			Original: r,
		}
		rec.SetLabel(r.Name, dc.Name)
		switch rtype := r.Type; rtype {
		case "ALIAS", "URL":
			rec.Type = r.Type
			rec.SetTarget(r.Content)
		case "MX":
			if err := rec.SetTargetMX(uint16(r.Priority), r.Content); err != nil {
				panic(errors.Wrap(err, "unparsable record received from dnsimple"))
			}
		default:
			if err := rec.PopulateFromString(r.Type, r.Content, dc.Name); err != nil {
				panic(errors.Wrap(err, "unparsable record received from dnsimple"))
			}
		}
		actual = append(actual, rec)
	}
	removeOtherNS(dc)
	// dc.Filter(func(r *models.RecordConfig) bool {
	// 	if r.Type == "CAA" || r.Type == "SRV" {
	// 		r.MergeToTarget()
	// 	}
	// 	return true
	// })

	// Normalize
	models.PostProcessRecords(actual)

	differ := diff.New(dc)
	_, create, delete, modify := differ.IncrementalDiff(actual)

	for _, del := range delete {
		rec := del.Existing.Original.(dnsimpleapi.ZoneRecord)
		corrections = append(corrections, &models.Correction{
			Msg: del.String(),
			F:   c.deleteRecordFunc(rec.ID, dc.Name),
		})
	}

	for _, cre := range create {
		rec := cre.Desired
		corrections = append(corrections, &models.Correction{
			Msg: cre.String(),
			F:   c.createRecordFunc(rec, dc.Name),
		})
	}

	for _, mod := range modify {
		old := mod.Existing.Original.(dnsimpleapi.ZoneRecord)
		new := mod.Desired
		corrections = append(corrections, &models.Correction{
			Msg: mod.String(),
			F:   c.updateRecordFunc(&old, new, dc.Name),
		})
	}

	return corrections, nil
}

// GetRegistrarCorrections returns corrections that update a domain's registrar.
func (c *DnsimpleApi) GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}

	nameServers, err := c.getNameservers(dc.Name)
	if err != nil {
		return nil, err
	}
  sort.Strings(nameServers)

	actual := strings.Join(nameServers, ",")

	expectedSet := []string{}
	for _, ns := range dc.Nameservers {
		expectedSet = append(expectedSet, ns.Name)
	}
	sort.Strings(expectedSet)
	expected := strings.Join(expectedSet, ",")

	if actual != expected {
		return []*models.Correction{
			{
				Msg: fmt.Sprintf("Update nameservers %s -> %s", actual, expected),
				F:   c.updateNameserversFunc(expectedSet, dc.Name),
			},
		}, nil
	}

	return corrections, nil
}

// DNSimple calls

func (c *DnsimpleApi) getClient() *dnsimpleapi.Client {
	client := dnsimpleapi.NewClient(dnsimpleapi.NewOauthTokenCredentials(c.AccountToken))
	if c.BaseURL != "" {
		client.BaseURL = c.BaseURL
	}
	return client
}

func (c *DnsimpleApi) getAccountID() (string, error) {
	if c.accountID == "" {
		client := c.getClient()
		whoamiResponse, err := client.Identity.Whoami()
		if err != nil {
			return "", err
		}
		if whoamiResponse.Data.User != nil && whoamiResponse.Data.Account == nil {
			return "", errors.Errorf("DNSimple token appears to be a user token. Please supply an account token")
		}
		c.accountID = strconv.Itoa(whoamiResponse.Data.Account.ID)
	}
	return c.accountID, nil
}

func (c *DnsimpleApi) getRecords(domainName string) ([]dnsimpleapi.ZoneRecord, error) {
	client := c.getClient()

	accountID, err := c.getAccountID()
	if err != nil {
		return nil, err
	}

	opts := &dnsimpleapi.ZoneRecordListOptions{}
	recs := []dnsimpleapi.ZoneRecord{}
	opts.Page = 1
	for {
		recordsResponse, err := client.Zones.ListRecords(accountID, domainName, opts)
		if err != nil {
			return nil, err
		}
		recs = append(recs, recordsResponse.Data...)
		pg := recordsResponse.Pagination
		if pg.CurrentPage == pg.TotalPages {
			break
		}
		opts.Page++
	}

	return recs, nil
}

// Returns the name server names that should be used. If the domain is registered
// then this method will return the delegation name servers. If this domain
// is hosted only, then it will return the default DNSimple name servers.
func (c *DnsimpleApi) getNameservers(domainName string) ([]string, error) {
	client := c.getClient()

	accountID, err := c.getAccountID()
	if err != nil {
		return nil, err
	}

	domainResponse, err := client.Domains.GetDomain(accountID, domainName)
	if err != nil {
		return nil, err
	}

	if domainResponse.Data.State == stateRegistered {

		delegationResponse, err := client.Registrar.GetDomainDelegation(accountID, domainName)
		if err != nil {
			return nil, err
		}

		return *delegationResponse.Data, nil
	}
	return defaultNameServerNames, nil
}

// Returns a function that can be invoked to change the delegation of the domain to the given name server names.
func (c *DnsimpleApi) updateNameserversFunc(nameServerNames []string, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		nameServers := dnsimpleapi.Delegation(nameServerNames)

		_, err = client.Registrar.ChangeDomainDelegation(accountID, domainName, &nameServers)
		if err != nil {
			return err
		}

		return nil
	}
}

// Returns a function that can be invoked to create a record in a zone.
func (c *DnsimpleApi) createRecordFunc(rc *models.RecordConfig, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}
		record := dnsimpleapi.ZoneRecord{
			Name:     rc.GetLabel(),
			Type:     rc.Type,
			Content:  rc.GetTargetCombined(),
			TTL:      int(rc.TTL),
			Priority: int(rc.MxPreference),
		}
		_, err = client.Zones.CreateRecord(accountID, domainName, record)
		if err != nil {
			return err
		}

		return nil
	}
}

// Returns a function that can be invoked to delete a record in a zone.
func (c *DnsimpleApi) deleteRecordFunc(recordID int, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		_, err = client.Zones.DeleteRecord(accountID, domainName, recordID)
		if err != nil {
			return err
		}

		return nil

	}
}

// Returns a function that can be invoked to update a record in a zone.
func (c *DnsimpleApi) updateRecordFunc(old *dnsimpleapi.ZoneRecord, rc *models.RecordConfig, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		record := dnsimpleapi.ZoneRecord{
			Name:     rc.GetLabel(),
			Type:     rc.Type,
			Content:  rc.GetTargetCombined(),
			TTL:      int(rc.TTL),
			Priority: int(rc.MxPreference),
		}

		_, err = client.Zones.UpdateRecord(accountID, domainName, old.ID, record)
		if err != nil {
			return err
		}

		return nil
	}
}

// constructors

func newReg(conf map[string]string) (providers.Registrar, error) {
	return newProvider(conf, nil)
}

func newDsp(conf map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	return newProvider(conf, metadata)
}

func newProvider(m map[string]string, metadata json.RawMessage) (*DnsimpleApi, error) {
	api := &DnsimpleApi{}
	api.AccountToken = m["token"]
	if api.AccountToken == "" {
		return nil, errors.Errorf("missing DNSimple token")
	}

	if m["baseurl"] != "" {
		api.BaseURL = m["baseurl"]
	}

	return api, nil
}

// remove all non-dnsimple NS records from our desired state.
// if any are found, print a warning
func removeOtherNS(dc *models.DomainConfig) {
	newList := make([]*models.RecordConfig, 0, len(dc.Records))
	for _, rec := range dc.Records {
		if rec.Type == "NS" {
			// apex NS inside dnsimple are expected.
			if rec.GetLabelFQDN() == dc.Name && strings.HasSuffix(rec.GetTargetField(), ".dnsimple.com.") {
				continue
			}
			fmt.Printf("Warning: dnsimple.com does not allow NS records to be modified. %s will not be added.\n", rec.GetTargetField())
			continue
		}
		newList = append(newList, rec)
	}
	dc.Records = newList
}
<commit_msg>dnsimple: gofmt (#388)<commit_after>package dnsimple

import (
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/providers"
	"github.com/StackExchange/dnscontrol/providers/diff"
	"github.com/pkg/errors"

	dnsimpleapi "github.com/dnsimple/dnsimple-go/dnsimple"
)

var features = providers.DocumentationNotes{
	providers.CanUseAlias:            providers.Can(),
	providers.CanUseCAA:              providers.Can(),
	providers.CanUsePTR:              providers.Can(),
	providers.CanUseSRV:              providers.Can(),
	providers.CanUseTLSA:             providers.Cannot(),
	providers.DocCreateDomains:       providers.Cannot(),
	providers.DocDualHost:            providers.Cannot("DNSimple does not allow sufficient control over the apex NS records"),
	providers.DocOfficiallySupported: providers.Cannot(),
}

func init() {
	providers.RegisterRegistrarType("DNSIMPLE", newReg)
	providers.RegisterDomainServiceProviderType("DNSIMPLE", newDsp, features)
}

const stateRegistered = "registered"

var defaultNameServerNames = []string{
	"ns1.dnsimple.com",
	"ns2.dnsimple.com",
	"ns3.dnsimple.com",
	"ns4.dnsimple.com",
}

// DnsimpleApi is the handle for this provider.
type DnsimpleApi struct {
	AccountToken string // The account access token
	BaseURL      string // An alternate base URI
	accountID    string // Account id cache
}

// GetNameservers returns the name servers for a domain.
func (c *DnsimpleApi) GetNameservers(domainName string) ([]*models.Nameserver, error) {
	return models.StringsToNameservers(defaultNameServerNames), nil
}

// GetDomainCorrections returns corrections that update a domain.
func (c *DnsimpleApi) GetDomainCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}
	dc.Punycode()
	records, err := c.getRecords(dc.Name)
	if err != nil {
		return nil, err
	}

	var actual []*models.RecordConfig
	for _, r := range records {
		if r.Type == "SOA" || r.Type == "NS" {
			continue
		}
		if r.Name == "" {
			r.Name = "@"
		}
		if r.Type == "CNAME" || r.Type == "MX" || r.Type == "ALIAS" || r.Type == "SRV" {
			r.Content += "."
		}
		// dnsimple adds these odd txt records that mirror the alias records.
		// they seem to manage them on deletes and things, so we'll just pretend they don't exist
		if r.Type == "TXT" && strings.HasPrefix(r.Content, "ALIAS for ") {
			continue
		}
		rec := &models.RecordConfig{
			TTL:      uint32(r.TTL),
			Original: r,
		}
		rec.SetLabel(r.Name, dc.Name)
		switch rtype := r.Type; rtype {
		case "ALIAS", "URL":
			rec.Type = r.Type
			rec.SetTarget(r.Content)
		case "MX":
			if err := rec.SetTargetMX(uint16(r.Priority), r.Content); err != nil {
				panic(errors.Wrap(err, "unparsable record received from dnsimple"))
			}
		default:
			if err := rec.PopulateFromString(r.Type, r.Content, dc.Name); err != nil {
				panic(errors.Wrap(err, "unparsable record received from dnsimple"))
			}
		}
		actual = append(actual, rec)
	}
	removeOtherNS(dc)
	// dc.Filter(func(r *models.RecordConfig) bool {
	// 	if r.Type == "CAA" || r.Type == "SRV" {
	// 		r.MergeToTarget()
	// 	}
	// 	return true
	// })

	// Normalize
	models.PostProcessRecords(actual)

	differ := diff.New(dc)
	_, create, delete, modify := differ.IncrementalDiff(actual)

	for _, del := range delete {
		rec := del.Existing.Original.(dnsimpleapi.ZoneRecord)
		corrections = append(corrections, &models.Correction{
			Msg: del.String(),
			F:   c.deleteRecordFunc(rec.ID, dc.Name),
		})
	}

	for _, cre := range create {
		rec := cre.Desired
		corrections = append(corrections, &models.Correction{
			Msg: cre.String(),
			F:   c.createRecordFunc(rec, dc.Name),
		})
	}

	for _, mod := range modify {
		old := mod.Existing.Original.(dnsimpleapi.ZoneRecord)
		new := mod.Desired
		corrections = append(corrections, &models.Correction{
			Msg: mod.String(),
			F:   c.updateRecordFunc(&old, new, dc.Name),
		})
	}

	return corrections, nil
}

// GetRegistrarCorrections returns corrections that update a domain's registrar.
func (c *DnsimpleApi) GetRegistrarCorrections(dc *models.DomainConfig) ([]*models.Correction, error) {
	corrections := []*models.Correction{}

	nameServers, err := c.getNameservers(dc.Name)
	if err != nil {
		return nil, err
	}
	sort.Strings(nameServers)

	actual := strings.Join(nameServers, ",")

	expectedSet := []string{}
	for _, ns := range dc.Nameservers {
		expectedSet = append(expectedSet, ns.Name)
	}
	sort.Strings(expectedSet)
	expected := strings.Join(expectedSet, ",")

	if actual != expected {
		return []*models.Correction{
			{
				Msg: fmt.Sprintf("Update nameservers %s -> %s", actual, expected),
				F:   c.updateNameserversFunc(expectedSet, dc.Name),
			},
		}, nil
	}

	return corrections, nil
}

// DNSimple calls

func (c *DnsimpleApi) getClient() *dnsimpleapi.Client {
	client := dnsimpleapi.NewClient(dnsimpleapi.NewOauthTokenCredentials(c.AccountToken))
	if c.BaseURL != "" {
		client.BaseURL = c.BaseURL
	}
	return client
}

func (c *DnsimpleApi) getAccountID() (string, error) {
	if c.accountID == "" {
		client := c.getClient()
		whoamiResponse, err := client.Identity.Whoami()
		if err != nil {
			return "", err
		}
		if whoamiResponse.Data.User != nil && whoamiResponse.Data.Account == nil {
			return "", errors.Errorf("DNSimple token appears to be a user token. Please supply an account token")
		}
		c.accountID = strconv.Itoa(whoamiResponse.Data.Account.ID)
	}
	return c.accountID, nil
}

func (c *DnsimpleApi) getRecords(domainName string) ([]dnsimpleapi.ZoneRecord, error) {
	client := c.getClient()

	accountID, err := c.getAccountID()
	if err != nil {
		return nil, err
	}

	opts := &dnsimpleapi.ZoneRecordListOptions{}
	recs := []dnsimpleapi.ZoneRecord{}
	opts.Page = 1
	for {
		recordsResponse, err := client.Zones.ListRecords(accountID, domainName, opts)
		if err != nil {
			return nil, err
		}
		recs = append(recs, recordsResponse.Data...)
		pg := recordsResponse.Pagination
		if pg.CurrentPage == pg.TotalPages {
			break
		}
		opts.Page++
	}

	return recs, nil
}

// Returns the name server names that should be used. If the domain is registered
// then this method will return the delegation name servers. If this domain
// is hosted only, then it will return the default DNSimple name servers.
func (c *DnsimpleApi) getNameservers(domainName string) ([]string, error) {
	client := c.getClient()

	accountID, err := c.getAccountID()
	if err != nil {
		return nil, err
	}

	domainResponse, err := client.Domains.GetDomain(accountID, domainName)
	if err != nil {
		return nil, err
	}

	if domainResponse.Data.State == stateRegistered {

		delegationResponse, err := client.Registrar.GetDomainDelegation(accountID, domainName)
		if err != nil {
			return nil, err
		}

		return *delegationResponse.Data, nil
	}
	return defaultNameServerNames, nil
}

// Returns a function that can be invoked to change the delegation of the domain to the given name server names.
func (c *DnsimpleApi) updateNameserversFunc(nameServerNames []string, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		nameServers := dnsimpleapi.Delegation(nameServerNames)

		_, err = client.Registrar.ChangeDomainDelegation(accountID, domainName, &nameServers)
		if err != nil {
			return err
		}

		return nil
	}
}

// Returns a function that can be invoked to create a record in a zone.
func (c *DnsimpleApi) createRecordFunc(rc *models.RecordConfig, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}
		record := dnsimpleapi.ZoneRecord{
			Name:     rc.GetLabel(),
			Type:     rc.Type,
			Content:  rc.GetTargetCombined(),
			TTL:      int(rc.TTL),
			Priority: int(rc.MxPreference),
		}
		_, err = client.Zones.CreateRecord(accountID, domainName, record)
		if err != nil {
			return err
		}

		return nil
	}
}

// Returns a function that can be invoked to delete a record in a zone.
func (c *DnsimpleApi) deleteRecordFunc(recordID int, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		_, err = client.Zones.DeleteRecord(accountID, domainName, recordID)
		if err != nil {
			return err
		}

		return nil

	}
}

// Returns a function that can be invoked to update a record in a zone.
func (c *DnsimpleApi) updateRecordFunc(old *dnsimpleapi.ZoneRecord, rc *models.RecordConfig, domainName string) func() error {
	return func() error {
		client := c.getClient()

		accountID, err := c.getAccountID()
		if err != nil {
			return err
		}

		record := dnsimpleapi.ZoneRecord{
			Name:     rc.GetLabel(),
			Type:     rc.Type,
			Content:  rc.GetTargetCombined(),
			TTL:      int(rc.TTL),
			Priority: int(rc.MxPreference),
		}

		_, err = client.Zones.UpdateRecord(accountID, domainName, old.ID, record)
		if err != nil {
			return err
		}

		return nil
	}
}

// constructors

func newReg(conf map[string]string) (providers.Registrar, error) {
	return newProvider(conf, nil)
}

func newDsp(conf map[string]string, metadata json.RawMessage) (providers.DNSServiceProvider, error) {
	return newProvider(conf, metadata)
}

func newProvider(m map[string]string, metadata json.RawMessage) (*DnsimpleApi, error) {
	api := &DnsimpleApi{}
	api.AccountToken = m["token"]
	if api.AccountToken == "" {
		return nil, errors.Errorf("missing DNSimple token")
	}

	if m["baseurl"] != "" {
		api.BaseURL = m["baseurl"]
	}

	return api, nil
}

// remove all non-dnsimple NS records from our desired state.
// if any are found, print a warning
func removeOtherNS(dc *models.DomainConfig) {
	newList := make([]*models.RecordConfig, 0, len(dc.Records))
	for _, rec := range dc.Records {
		if rec.Type == "NS" {
			// apex NS inside dnsimple are expected.
			if rec.GetLabelFQDN() == dc.Name && strings.HasSuffix(rec.GetTargetField(), ".dnsimple.com.") {
				continue
			}
			fmt.Printf("Warning: dnsimple.com does not allow NS records to be modified. %s will not be added.\n", rec.GetTargetField())
			continue
		}
		newList = append(newList, rec)
	}
	dc.Records = newList
}
<|endoftext|>


<commit_before>// Copyright 2016 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"neugram.io/ng/eval"
	"neugram.io/ng/eval/environ"
	"neugram.io/ng/eval/shell"
	"neugram.io/ng/format"
	"neugram.io/ng/parser"
	"neugram.io/ng/tipe"

	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	historyShFile = ""
	historySh     = make(chan string, 1)
	sigint        = make(chan os.Signal, 1)

	p   *parser.Parser
	prg *eval.Program
)

func exit(code int) {
	if lineNg != nil {
		lineNg.Close()
	}
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format+"\n", args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

const usageLine = "ng [programfile | -e cmd] [arguments]"

func usage() {
	fmt.Fprintf(os.Stderr, `ng - neugram scripting language and shell

Usage:
	%s

Options:
`, usageLine)
	flag.PrintDefaults()
}

func main() {
	shell.Init()

	help := flag.Bool("h", false, "display help message and exit")
	e := flag.String("e", "", "program passed as a string")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s\n", usageLine)
		os.Exit(1)
	}
	flag.Parse()

	if *help {
		usage()
		os.Exit(0)
	}
	if *e != "" {
		initProgram(filepath.Join(cwd, "ng-arg"))
		res := p.ParseLine([]byte(*e))
		handleResult(res)
		return
	}
	if args := flag.Args(); len(args) > 0 {
		// TODO: plumb through the rest of the args
		path := args[0]
		initProgram(path)
		f, err := os.Open(path)
		if err != nil {
			exitf("%v", err)
		}
		state, err := runFile(f)
		if err != nil {
			exitf("%v", err)
		}
		if state == parser.StateCmd {
			exitf("%s: ends in an unclosed shell statement", args[0])
		}
		return
	}

	origMode = mode()
	lineNg = liner.NewLiner()
	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func ps1(env *environ.Environ) string {
	v := env.Get("PS1")
	if v == "" {
		return "ng$ "
	}
	if strings.IndexByte(v, '\\') == -1 {
		return v
	}
	var buf []byte
	for {
		i := strings.IndexByte(v, '\\')
		if i == -1 || i == len(v)-1 {
			break
		}
		buf = append(buf, v[:i]...)
		b := v[i+1]
		v = v[i+2:]
		switch b {
		case 'h', 'H':
			out, err := exec.Command("hostname").CombinedOutput()
			if err != nil {
				fmt.Fprintf(os.Stderr, "ng: %v\n", err)
				continue
			}
			if b == 'h' {
				if i := bytes.IndexByte(out, '.'); i >= 0 {
					out = out[:i]
				}
			}
			if len(out) > 0 && out[len(out)-1] == '\n' {
				out = out[:len(out)-1]
			}
			buf = append(buf, out...)
		case 'n':
			buf = append(buf, '\n')
		case 'w', 'W':
			cwd := env.Get("PWD")
			if home := env.Get("HOME"); home != "" {
				cwd = strings.Replace(cwd, home, "~", 1)
			}
			if b == 'W' {
				cwd = filepath.Base(cwd)
			}
			buf = append(buf, cwd...)
		}
		// TODO: '!', '#', '$', 'nnn', 's', 'j', and more.
	}
	buf = append(buf, v...)
	return string(buf)
}

var cwd string

func init() {
	var err error
	cwd, err = os.Getwd()
	if err != nil {
		panic(err)
	}
}

func initProgram(path string) {
	p = parser.New()
	prg = eval.New(path)
	shell.Env = prg.Environ()
	shell.Alias = prg.Alias()

	// TODO this env setup could be done in neugram code
	env := prg.Environ()
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env.Set(s[:i], s[i+1:])
	}
	wd, err := os.Getwd()
	if err == nil {
		env.Set("PWD", wd)
	}
	//setWindowSize(env)

	signal.Notify(sigint, os.Interrupt)
}

func runFile(f *os.File) (parser.ParserState, error) {
	state := parser.StateStmt
	scanner := bufio.NewScanner(f)
	for i := 0; scanner.Scan(); i++ {
		b := scanner.Bytes()
		if i == 0 && len(b) > 2 && b[0] == '#' && b[1] == '!' { // shebang
			continue
		}
		res := p.ParseLine(b)
		handleResult(res)
		state = res.State
	}
	if err := scanner.Err(); err != nil {
		return state, fmt.Errorf("%s: %v", f.Name(), err)
	}
	switch state {
	case parser.StateStmtPartial, parser.StateCmdPartial:
		return state, fmt.Errorf("%s: ends in a partial statement", f.Name())
	default:
		return state, nil
	}
}

func loop() {
	path := filepath.Join(cwd, "ng-interactive")
	initProgram(path)

	state := parser.StateStmt
	if os.Args[0] == "ngsh" || os.Args[0] == "-ngsh" {
		initFile := filepath.Join(os.Getenv("HOME"), ".ngshinit")
		if f, err := os.Open(initFile); err == nil {
			var err error
			state, err = runFile(f)
			f.Close()
			if err != nil {
				exitf("%v", err)
			}
		}
		if state == parser.StateStmt {
			res := p.ParseLine([]byte("$$"))
			handleResult(res)
			state = res.State
		}
	}

	lineNg.SetTabCompletionStyle(liner.TabPrints)
	lineNg.SetWordCompleter(completer)
	lineNg.SetCtrlCAborts(true)

	if f, err := os.Open(historyShFile); err == nil {
		lineNg.SetMode("sh")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.SetMode("ng")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	for {
		var (
			mode    string
			prompt  string
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			mode, prompt, history = "ng", "??> ", historyNg
		case parser.StateStmt:
			mode, prompt, history = "ng", "ng> ", historyNg
		case parser.StateStmtPartial:
			mode, prompt, history = "ng", "..> ", historyNg
		case parser.StateCmd:
			mode, prompt, history = "sh", ps1(prg.Environ()), historySh
		case parser.StateCmdPartial:
			mode, prompt, history = "sh", "..$ ", historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		lineNg.SetMode(mode)
		data, err := lineNg.Prompt(prompt)
		if err == liner.ErrPromptAborted {
			switch state {
			case parser.StateStmtPartial:
				fmt.Printf("TODO interrupt partial statement\n")
			case parser.StateCmdPartial:
				fmt.Printf("TODO interrupt partial command\n")
			}
		} else if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		lineNg.AppendHistory(mode, data)
		history <- data
		select { // drain sigint
		case <-sigint:
		default:
		}
		res := p.ParseLine([]byte(data))
		handleResult(res)
		state = res.State
	}
}

func handleResult(res parser.Result) {
	for _, s := range res.Stmts {
		v, err := prg.Eval(s, sigint)
		if err != nil {
			fmt.Printf("ng: %v\n", err)
			continue
		}
		if len(v) > 1 {
			fmt.Print("(")
		}
		for i, val := range v {
			if i > 0 {
				fmt.Print(", ")
			}
			if val == (reflect.Value{}) {
				fmt.Print("<nil>")
				continue
			}
			switch v := val.Interface().(type) {
			case eval.UntypedInt:
				fmt.Print(v.String())
			case eval.UntypedFloat:
				fmt.Print(v.String())
			case eval.UntypedString:
				fmt.Print(v.String)
			case eval.UntypedRune:
				fmt.Print("%s", v.Rune)
			case eval.UntypedBool:
				fmt.Print(v.Bool)
			default:
				fmt.Print(format.Debug(v))
			}
		}
		if len(v) > 1 {
			fmt.Println(")")
		} else if len(v) == 1 {
			fmt.Println("")
		}
	}
	for _, err := range res.Errs {
		fmt.Println(err.Error())
	}
	//editMode := mode()
	//origMode.ApplyMode()
	for _, cmd := range res.Cmds {
		j := &shell.Job{
			Cmd:    cmd,
			Params: prg,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}
		if err := j.Start(); err != nil {
			fmt.Println(err)
			continue
		}
		done, err := j.Wait()
		if err != nil {
			fmt.Println(err)
			continue
		}
		if !done {
			break // TODO not right, instead we should just have one cmd, not Cmds here.
		}
	}
	//editMode.ApplyMode()
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	/*switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
	fmt.Print("{")
	for i, name := range t.FieldNames {
		fmt.Printf("%s: ", name)
		printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
		if i < len(t.FieldNames)-1 {
			fmt.Print(", ")
		}
	}
	fmt.Print("}")
	default:
	}*/
	fmt.Print(v)
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ngsh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				// TODO: FcntlFlock
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<commit_msg>ng: properly restore terminal state<commit_after>// Copyright 2016 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"reflect"
	"strings"
	"time"

	"neugram.io/ng/eval"
	"neugram.io/ng/eval/environ"
	"neugram.io/ng/eval/shell"
	"neugram.io/ng/format"
	"neugram.io/ng/parser"
	"neugram.io/ng/tipe"

	"github.com/peterh/liner"
)

var (
	origMode liner.ModeApplier

	lineNg        *liner.State // ng-mode line reader
	historyNgFile = ""
	historyNg     = make(chan string, 1)
	historyShFile = ""
	historySh     = make(chan string, 1)
	sigint        = make(chan os.Signal, 1)

	p   *parser.Parser
	prg *eval.Program
)

func exit(code int) {
	if lineNg != nil {
		lineNg.Close()
	}
	os.Exit(code)
}

func exitf(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "ng: "+format+"\n", args...)
	exit(1)
}

func mode() liner.ModeApplier {
	m, err := liner.TerminalMode()
	if err != nil {
		exitf("terminal mode: %v", err)
	}
	return m
}

const usageLine = "ng [programfile | -e cmd] [arguments]"

func usage() {
	fmt.Fprintf(os.Stderr, `ng - neugram scripting language and shell

Usage:
	%s

Options:
`, usageLine)
	flag.PrintDefaults()
}

func main() {
	shell.Init()

	help := flag.Bool("h", false, "display help message and exit")
	e := flag.String("e", "", "program passed as a string")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s\n", usageLine)
		os.Exit(1)
	}
	flag.Parse()

	if *help {
		usage()
		os.Exit(0)
	}
	if *e != "" {
		initProgram(filepath.Join(cwd, "ng-arg"))
		res := p.ParseLine([]byte(*e))
		handleResult(res)
		return
	}
	if args := flag.Args(); len(args) > 0 {
		// TODO: plumb through the rest of the args
		path := args[0]
		initProgram(path)
		f, err := os.Open(path)
		if err != nil {
			exitf("%v", err)
		}
		state, err := runFile(f)
		if err != nil {
			exitf("%v", err)
		}
		if state == parser.StateCmd {
			exitf("%s: ends in an unclosed shell statement", args[0])
		}
		return
	}

	origMode = mode()
	lineNg = liner.NewLiner()
	defer lineNg.Close()

	loop()
}

func setWindowSize(env map[interface{}]interface{}) {
	// TODO windowsize
	// TODO
	// TODO
	// TODO
	/*
		rows, cols, err := job.WindowSize(os.Stderr.Fd())
		if err != nil {
			fmt.Printf("ng: could not get window size: %v\n", err)
		} else {
			// TODO: these are meant to be shell variables, not
			// environment variables. But then, how do programs
			// like `ls` read them?
			env["LINES"] = strconv.Itoa(rows)
			env["COLUMNS"] = strconv.Itoa(cols)
		}
	*/
}

func ps1(env *environ.Environ) string {
	v := env.Get("PS1")
	if v == "" {
		return "ng$ "
	}
	if strings.IndexByte(v, '\\') == -1 {
		return v
	}
	var buf []byte
	for {
		i := strings.IndexByte(v, '\\')
		if i == -1 || i == len(v)-1 {
			break
		}
		buf = append(buf, v[:i]...)
		b := v[i+1]
		v = v[i+2:]
		switch b {
		case 'h', 'H':
			out, err := exec.Command("hostname").CombinedOutput()
			if err != nil {
				fmt.Fprintf(os.Stderr, "ng: %v\n", err)
				continue
			}
			if b == 'h' {
				if i := bytes.IndexByte(out, '.'); i >= 0 {
					out = out[:i]
				}
			}
			if len(out) > 0 && out[len(out)-1] == '\n' {
				out = out[:len(out)-1]
			}
			buf = append(buf, out...)
		case 'n':
			buf = append(buf, '\n')
		case 'w', 'W':
			cwd := env.Get("PWD")
			if home := env.Get("HOME"); home != "" {
				cwd = strings.Replace(cwd, home, "~", 1)
			}
			if b == 'W' {
				cwd = filepath.Base(cwd)
			}
			buf = append(buf, cwd...)
		}
		// TODO: '!', '#', '$', 'nnn', 's', 'j', and more.
	}
	buf = append(buf, v...)
	return string(buf)
}

var cwd string

func init() {
	var err error
	cwd, err = os.Getwd()
	if err != nil {
		panic(err)
	}
}

func initProgram(path string) {
	p = parser.New()
	prg = eval.New(path)
	shell.Env = prg.Environ()
	shell.Alias = prg.Alias()

	// TODO this env setup could be done in neugram code
	env := prg.Environ()
	for _, s := range os.Environ() {
		i := strings.Index(s, "=")
		env.Set(s[:i], s[i+1:])
	}
	wd, err := os.Getwd()
	if err == nil {
		env.Set("PWD", wd)
	}
	//setWindowSize(env)

	signal.Notify(sigint, os.Interrupt)
}

func runFile(f *os.File) (parser.ParserState, error) {
	state := parser.StateStmt
	scanner := bufio.NewScanner(f)
	for i := 0; scanner.Scan(); i++ {
		b := scanner.Bytes()
		if i == 0 && len(b) > 2 && b[0] == '#' && b[1] == '!' { // shebang
			continue
		}
		res := p.ParseLine(b)
		handleResult(res)
		state = res.State
	}
	if err := scanner.Err(); err != nil {
		return state, fmt.Errorf("%s: %v", f.Name(), err)
	}
	switch state {
	case parser.StateStmtPartial, parser.StateCmdPartial:
		return state, fmt.Errorf("%s: ends in a partial statement", f.Name())
	default:
		return state, nil
	}
}

func loop() {
	path := filepath.Join(cwd, "ng-interactive")
	initProgram(path)

	state := parser.StateStmt
	if os.Args[0] == "ngsh" || os.Args[0] == "-ngsh" {
		initFile := filepath.Join(os.Getenv("HOME"), ".ngshinit")
		if f, err := os.Open(initFile); err == nil {
			var err error
			state, err = runFile(f)
			f.Close()
			if err != nil {
				exitf("%v", err)
			}
		}
		if state == parser.StateStmt {
			res := p.ParseLine([]byte("$$"))
			handleResult(res)
			state = res.State
		}
	}

	lineNg.SetTabCompletionStyle(liner.TabPrints)
	lineNg.SetWordCompleter(completer)
	lineNg.SetCtrlCAborts(true)

	if f, err := os.Open(historyShFile); err == nil {
		lineNg.SetMode("sh")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyShFile, historySh)

	if f, err := os.Open(historyNgFile); err == nil {
		lineNg.SetMode("ng")
		lineNg.ReadHistory(f)
		f.Close()
	}
	go historyWriter(historyNgFile, historyNg)

	for {
		var (
			mode    string
			prompt  string
			history chan string
		)
		switch state {
		case parser.StateUnknown:
			mode, prompt, history = "ng", "??> ", historyNg
		case parser.StateStmt:
			mode, prompt, history = "ng", "ng> ", historyNg
		case parser.StateStmtPartial:
			mode, prompt, history = "ng", "..> ", historyNg
		case parser.StateCmd:
			mode, prompt, history = "sh", ps1(prg.Environ()), historySh
		case parser.StateCmdPartial:
			mode, prompt, history = "sh", "..$ ", historySh
		default:
			exitf("unkown parser state: %v", state)
		}
		lineNg.SetMode(mode)
		data, err := lineNg.Prompt(prompt)
		if err == liner.ErrPromptAborted {
			switch state {
			case parser.StateStmtPartial:
				fmt.Printf("TODO interrupt partial statement\n")
			case parser.StateCmdPartial:
				fmt.Printf("TODO interrupt partial command\n")
			}
		} else if err != nil {
			if err == io.EOF {
				exit(0)
			}
			exitf("error reading input: %v", err)
		}
		if data == "" {
			continue
		}
		lineNg.AppendHistory(mode, data)
		history <- data
		select { // drain sigint
		case <-sigint:
		default:
		}
		res := p.ParseLine([]byte(data))
		handleResult(res)
		state = res.State
	}
}

func handleResult(res parser.Result) {
	for _, s := range res.Stmts {
		v, err := prg.Eval(s, sigint)
		if err != nil {
			fmt.Printf("ng: %v\n", err)
			continue
		}
		if len(v) > 1 {
			fmt.Print("(")
		}
		for i, val := range v {
			if i > 0 {
				fmt.Print(", ")
			}
			if val == (reflect.Value{}) {
				fmt.Print("<nil>")
				continue
			}
			switch v := val.Interface().(type) {
			case eval.UntypedInt:
				fmt.Print(v.String())
			case eval.UntypedFloat:
				fmt.Print(v.String())
			case eval.UntypedString:
				fmt.Print(v.String)
			case eval.UntypedRune:
				fmt.Print("%s", v.Rune)
			case eval.UntypedBool:
				fmt.Print(v.Bool)
			default:
				fmt.Print(format.Debug(v))
			}
		}
		if len(v) > 1 {
			fmt.Println(")")
		} else if len(v) == 1 {
			fmt.Println("")
		}
	}
	for _, err := range res.Errs {
		fmt.Println(err.Error())
	}
	//editMode := mode()
	//origMode.ApplyMode()
	for _, cmd := range res.Cmds {
		j := &shell.Job{
			Cmd:    cmd,
			Params: prg,
			Stdin:  os.Stdin,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}
		if err := j.Start(); err != nil {
			fmt.Println(err)
			continue
		}
		done, err := j.Wait()
		if err != nil {
			fmt.Println(err)
			continue
		}
		if !done {
			break // TODO not right, instead we should just have one cmd, not Cmds here.
		}
	}
	//editMode.ApplyMode()
}

func printValue(t tipe.Type, v interface{}) {
	// This is, effectively, a primitive type-aware printf implementation
	// that understands the neugram evaluator data layout. A far better
	// version of this would be an "ngfmt" package, that implemented the
	// printing command in neugram, using a "ngreflect" package. But it
	// will be a while until I build a reflect package, so this will have
	// to do.
	//
	// Still: avoid putting too much machinary in this. At some point soon
	// it's not worth the effort.
	/*switch t := tipe.Underlying(t).(type) {
	case *tipe.Struct:
	fmt.Print("{")
	for i, name := range t.FieldNames {
		fmt.Printf("%s: ", name)
		printValue(t.Fields[i], v.(*eval.StructVal).Fields[i].Value)
		if i < len(t.FieldNames)-1 {
			fmt.Print(", ")
		}
	}
	fmt.Print("}")
	default:
	}*/
	fmt.Print(v)
}

func init() {
	if home := os.Getenv("HOME"); home != "" {
		historyNgFile = filepath.Join(home, ".ng_history")
		historyShFile = filepath.Join(home, ".ngsh_history")
	}
}

func historyWriter(dst string, src <-chan string) {
	var batch []string
	ticker := time.Tick(250 * time.Millisecond)
	for {
		select {
		case line := <-src:
			batch = append(batch, line)
		case <-ticker:
			if len(batch) > 0 && dst != "" {
				// TODO: FcntlFlock
				f, err := os.OpenFile(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0664)
				if err == nil {
					for _, line := range batch {
						fmt.Fprintf(f, "%s\n", line)
					}
					f.Close()
				}
			}
			batch = nil
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"
	"strings"

	"veyron/examples/tunnel"
	"veyron/examples/tunnel/tunneld/impl"
	"veyron/lib/signals"
	"veyron2/ipc"
	"veyron2/rt"
	"veyron2/security"
	"veyron2/vlog"
)

var (
	// TODO(rthellend): Remove the address and protocol flags when the config manager is working.
	protocol = flag.String("protocol", "tcp", "network to listen on. For example, set to 'veyron' and set --address to the endpoint/name of a proxy to have this tunnel service proxied.")
	address  = flag.String("address", ":0", "address to listen on")

	users = flag.String("users", "", "A comma-separated list of principal patterns allowed to use this service.")
)

// firstHardwareAddrInUse returns the hwaddr of the first network interface
// that is up, excluding loopback.
func firstHardwareAddrInUse() (string, error) {
	interfaces, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for _, i := range interfaces {
		if i.Name != "lo" && i.Flags&net.FlagUp != 0 {
			name := i.HardwareAddr.String()
			vlog.Infof("Using %q (from %v)", name, i.Name)
			return name, nil
		}
	}
	return "", errors.New("No usable network interfaces")
}

func authorizer() security.Authorizer {
	ACL := make(security.ACL)
	principals := strings.Split(*users, ",")
	for _, p := range principals {
		ACL[security.PrincipalPattern(p)] = security.LabelSet(security.AdminLabel)
	}
	return security.NewACLAuthorizer(ACL)
}

func main() {
	r := rt.Init()
	defer r.Shutdown()
	server, err := r.NewServer()
	if err != nil {
		vlog.Fatalf("NewServer failed: %v", err)
	}
	defer server.Stop()

	if err := server.Register("", ipc.SoloDispatcher(tunnel.NewServerTunnel(&impl.T{}), authorizer())); err != nil {
		vlog.Fatalf("Register failed: %v", err)
	}
	ep, err := server.Listen(*protocol, *address)
	if err != nil {
		vlog.Fatalf("Listen(%q, %q) failed: %v", "tcp", *address, err)
	}
	hwaddr, err := firstHardwareAddrInUse()
	if err != nil {
		vlog.Fatalf("Couldn't find a good hw address: %v", err)
	}
	hostname, err := os.Hostname()
	if err != nil {
		vlog.Fatalf("os.Hostname failed: %v", err)
	}
	// TODO(rthellend): This is not secure. We should use
	// rt.R().Product().ID() and the associated verification, when it is
	// ready.
	names := []string{
		fmt.Sprintf("tunnel/hostname/%s", hostname),
		fmt.Sprintf("tunnel/hwaddr/%s", hwaddr),
		fmt.Sprintf("tunnel/id/%s", rt.R().Identity().PublicID()),
	}
	published := false
	for _, n := range names {
		if err := server.Publish(n); err != nil {
			vlog.Infof("Publish(%v) failed: %v", n, err)
			continue
		}
		published = true
	}
	if !published {
		vlog.Fatalf("Failed to publish with any of %v", names)
	}
	vlog.Infof("Listening on endpoint /%s (published as %v)", ep, names)
	<-signals.ShutdownOnSignals()
}
<commit_msg>veyron/examples/{tunnel,rockpaperscissors}: Use the library to generate ACL authorizers instead of using custom flags.<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"net"
	"os"

	"veyron/examples/tunnel"
	"veyron/examples/tunnel/tunneld/impl"
	"veyron/lib/signals"
	sflag "veyron/security/flag"
	"veyron2/ipc"
	"veyron2/rt"
	"veyron2/vlog"
)

var (
	// TODO(rthellend): Remove the address and protocol flags when the config manager is working.
	protocol = flag.String("protocol", "tcp", "network to listen on. For example, set to 'veyron' and set --address to the endpoint/name of a proxy to have this tunnel service proxied.")
	address  = flag.String("address", ":0", "address to listen on")
)

// firstHardwareAddrInUse returns the hwaddr of the first network interface
// that is up, excluding loopback.
func firstHardwareAddrInUse() (string, error) {
	interfaces, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for _, i := range interfaces {
		if i.Name != "lo" && i.Flags&net.FlagUp != 0 {
			name := i.HardwareAddr.String()
			vlog.Infof("Using %q (from %v)", name, i.Name)
			return name, nil
		}
	}
	return "", errors.New("No usable network interfaces")
}

func main() {
	r := rt.Init()
	defer r.Shutdown()
	server, err := r.NewServer()
	if err != nil {
		vlog.Fatalf("NewServer failed: %v", err)
	}
	defer server.Stop()

	if err := server.Register("", ipc.SoloDispatcher(tunnel.NewServerTunnel(&impl.T{}), sflag.NewAuthorizerOrDie())); err != nil {
		vlog.Fatalf("Register failed: %v", err)
	}
	ep, err := server.Listen(*protocol, *address)
	if err != nil {
		vlog.Fatalf("Listen(%q, %q) failed: %v", "tcp", *address, err)
	}
	hwaddr, err := firstHardwareAddrInUse()
	if err != nil {
		vlog.Fatalf("Couldn't find a good hw address: %v", err)
	}
	hostname, err := os.Hostname()
	if err != nil {
		vlog.Fatalf("os.Hostname failed: %v", err)
	}
	// TODO(rthellend): This is not secure. We should use
	// rt.R().Product().ID() and the associated verification, when it is
	// ready.
	names := []string{
		fmt.Sprintf("tunnel/hostname/%s", hostname),
		fmt.Sprintf("tunnel/hwaddr/%s", hwaddr),
		fmt.Sprintf("tunnel/id/%s", rt.R().Identity().PublicID()),
	}
	published := false
	for _, n := range names {
		if err := server.Publish(n); err != nil {
			vlog.Infof("Publish(%v) failed: %v", n, err)
			continue
		}
		published = true
	}
	if !published {
		vlog.Fatalf("Failed to publish with any of %v", names)
	}
	vlog.Infof("Listening on endpoint /%s (published as %v)", ep, names)
	<-signals.ShutdownOnSignals()
}
<|endoftext|>