<commit_before>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var _ = Describe("Push", func() {
	root := filepath.Join(os.TempDir(), "PushTest")
	originRoot := filepath.Join(os.TempDir(), "PushOriginTest")
	forkRoot := filepath.Join(os.TempDir(), "PushForkTest")
	originBinStore := filepath.Join(os.TempDir(), "PushOriginBinStoreTest")
	forkBinStore := filepath.Join(os.TempDir(), "PushForkBinStoreTest")
	var oldwd string

	masterfilespercommit := [][]string{
		[]string{
			"img1.png", "img2.jpg",
			filepath.Join("movies", "movie1.mov"),
			filepath.Join("movies", "movie2.mov"),
			filepath.Join("other", "files", "windows.bmp"),
		},
		[]string{
			"img2.tga", "img3.tiff", "img4.png",
			filepath.Join("movies", "movie2.mov"),
			filepath.Join("movies", "movie3.mov"),
			filepath.Join("other", "files", "windows7.bmp"),
		},
		[]string{
			"img6.jpg",
			filepath.Join("other", "files", "windows.bmp"),
		},
	}
	branch2filespercommit := [][]string{
		[]string{
			"img4.png", "img5.jpg",
			filepath.Join("movies", "movie3.mov"),
		},
		[]string{
			"img7.jpg",
			filepath.Join("other", "files", "windows8.bmp"),
		},
	}
	sizeForFile := func(filename string, i int) int64 {
		// not actually that big, we're not doing size tests here
		if strings.HasSuffix(filename, ".mov") {
			return int64(i%3*1000 + 2000)
		} else {
			return int64(i%3*100 + 300)
		}
	}
	var mastershaspercommit [][]string
	var branch2shaspercommit [][]string
	checkLOBsExist := func(shas []string, path string) {
		for _, sha := range shas {
			meta := filepath.Join(path, getLOBMetaRelativePath(sha))
			_, err := os.Stat(meta)
			Expect(err).To(BeNil(), "Meta file should exist")
			// Assuming only one chunk for this test
			chunk := filepath.Join(path, getLOBChunkRelativePath(sha, 0))
			_, err = os.Stat(chunk)
			Expect(err).To(BeNil(), "Chunk file should exist")
		}

	}
	BeforeEach(func() {
		CreateGitRepoForTest(root)
		oldwd, _ = os.Getwd()
		os.Chdir(root)

		// Create 2 remotes
		CreateBareGitRepoForTest(originRoot)
		CreateBareGitRepoForTest(forkRoot)
		os.MkdirAll(originBinStore, 0755)
		os.MkdirAll(forkBinStore, 0755)

		// Make a file:// ref so we don't have hardlinks (more standard)
		originPathUrl := strings.Replace(originRoot, "\\", "/", -1)
		originPathUrl = "file://" + originPathUrl
		forkPathUrl := strings.Replace(forkRoot, "\\", "/", -1)
		forkPathUrl = "file://" + forkPathUrl
		f, err := os.OpenFile(filepath.Join(".git", "config"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		Expect(err).To(BeNil(), "Should not error trying to open config file")
		f.WriteString(fmt.Sprintf(`[remote "origin"]
    url = %v
    fetch = +refs/heads/*:refs/remotes/origin/*
    git-lob-path = %v
    git-lob-provider = filesystem
[remote "fork"]
    url = %v
    fetch = +refs/heads/*:refs/remotes/fork/*
    git-lob-path = %v
    git-lob-provider = filesystem
`, originPathUrl, originBinStore, forkPathUrl, forkBinStore))
		f.Close()

		LoadConfig(GlobalOptions)
		InitCoreProviders()

		// Create 3 commits with binary file references on master
		mastershaspercommit = CreateManyCommitsForTest(masterfilespercommit, 0, sizeForFile)
		// create another branch, from Tag1 which previous call created on 2nd commit
		err = exec.Command("git", "checkout", "-b", "branch2", "Tag1").Run()
		Expect(err).To(BeNil(), "Didn't create branch")
		// Create 2 more commits on this branch
		branch2shaspercommit = CreateManyCommitsForTest(branch2filespercommit, 3, sizeForFile)
		// Go back
		err = exec.Command("git", "checkout", "master").Run()
		Expect(err).To(BeNil(), "Didn't checkout master")
		// Note that working copy won't have correct binary data because filters aren't necessarily configured
		// see integration_test.go for those tests

	})
	AfterEach(func() {
		os.Chdir(oldwd)
		os.RemoveAll(root)
		os.RemoveAll(originRoot)
		os.RemoveAll(forkRoot)
		// Reset any option changes
		GlobalOptions = NewOptions()
	})

	It("Pushes correctly (Basic)", func() {
		originprovider, err := GetProviderForRemote("origin")
		Expect(err).To(BeNil(), "Shouldn't be an issue getting provider")

		var filesTransferred int
		var filesSkipped int
		var filesFailed int
		callback := func(data *ProgressCallbackData) (abort bool) {
			switch data.Type {
			case ProgressTransferBytes:
				if data.ItemBytesDone == data.ItemBytes {
					filesTransferred++
				}
			case ProgressSkip:
				filesSkipped++
			case ProgressError:
				filesFailed++
			case ProgressNotFound:
				filesFailed++
			}
			return false
		}
		// Start by pushing up to Tag1 so that we push only first 2 commits on master
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "Tag1"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount := (len(masterfilespercommit[0]) + len(masterfilespercommit[1])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped")
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[0], originBinStore)
		checkLOBsExist(mastershaspercommit[1], originBinStore)

		// Now push all of master, should skip previous & upload new
		filesTransferred = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = len(masterfilespercommit[2]) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[2], originBinStore)

		// Now push a different branch
		// Now push all of master, should skip previous & upload new
		filesTransferred = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "branch2"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = (len(branch2filespercommit[0]) + len(branch2filespercommit[1])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		// Confirm data exists on remote
		checkLOBsExist(branch2shaspercommit[0], originBinStore)
		checkLOBsExist(branch2shaspercommit[1], originBinStore)

		// Now push master to fork
		filesTransferred = 0
		err = PushBasic(originprovider, "fork", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = (len(masterfilespercommit[0]) + len(masterfilespercommit[1]) + len(masterfilespercommit[2])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[0], forkBinStore)
		checkLOBsExist(mastershaspercommit[1], forkBinStore)

	})

})
<commit_msg>Add test cases for pushing when data is not found, and test push cache marker is advanced correctly<commit_after>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var _ = Describe("Push", func() {
	root := filepath.Join(os.TempDir(), "PushTest")
	originRoot := filepath.Join(os.TempDir(), "PushOriginTest")
	forkRoot := filepath.Join(os.TempDir(), "PushForkTest")
	originBinStore := filepath.Join(os.TempDir(), "PushOriginBinStoreTest")
	forkBinStore := filepath.Join(os.TempDir(), "PushForkBinStoreTest")
	var oldwd string

	masterfilespercommit := [][]string{
		[]string{
			"img1.png", "img2.jpg",
			filepath.Join("movies", "movie1.mov"),
			filepath.Join("movies", "movie2.mov"),
			filepath.Join("other", "files", "windows.bmp"),
		},
		[]string{
			"img2.tga", "img3.tiff", "img4.png",
			filepath.Join("movies", "movie2.mov"),
			filepath.Join("movies", "movie3.mov"),
			filepath.Join("other", "files", "windows7.bmp"),
		},
		[]string{
			"img6.jpg",
			filepath.Join("other", "files", "windows.bmp"),
		},
	}
	branch2filespercommit := [][]string{
		[]string{
			"img4.png", "img5.jpg",
			filepath.Join("movies", "movie3.mov"),
		},
		[]string{
			"img7.jpg",
			filepath.Join("other", "files", "windows8.bmp"),
		},
	}
	sizeForFile := func(filename string, i int) int64 {
		// not actually that big, we're not doing size tests here
		if strings.HasSuffix(filename, ".mov") {
			return int64(i%3*1000 + 2000)
		} else {
			return int64(i%3*100 + 300)
		}
	}
	var mastershaspercommit [][]string
	var branch2shaspercommit [][]string
	checkLOBsExist := func(shas []string, path string) {
		for _, sha := range shas {
			meta := filepath.Join(path, getLOBMetaRelativePath(sha))
			_, err := os.Stat(meta)
			Expect(err).To(BeNil(), "Meta file should exist")
			// Assuming only one chunk for this test
			chunk := filepath.Join(path, getLOBChunkRelativePath(sha, 0))
			_, err = os.Stat(chunk)
			Expect(err).To(BeNil(), "Chunk file should exist")
		}

	}
	removeLOBs := func(shas []string, path string) {
		for _, sha := range shas {
			meta := filepath.Join(path, getLOBMetaRelativePath(sha))
			os.Remove(meta)
			chunk := filepath.Join(path, getLOBChunkRelativePath(sha, 0))
			os.Remove(chunk)
		}

	}
	BeforeEach(func() {
		CreateGitRepoForTest(root)
		oldwd, _ = os.Getwd()
		os.Chdir(root)

		// Create 2 remotes
		CreateBareGitRepoForTest(originRoot)
		CreateBareGitRepoForTest(forkRoot)
		os.MkdirAll(originBinStore, 0755)
		os.MkdirAll(forkBinStore, 0755)

		// Make a file:// ref so we don't have hardlinks (more standard)
		originPathUrl := strings.Replace(originRoot, "\\", "/", -1)
		originPathUrl = "file://" + originPathUrl
		forkPathUrl := strings.Replace(forkRoot, "\\", "/", -1)
		forkPathUrl = "file://" + forkPathUrl
		f, err := os.OpenFile(filepath.Join(".git", "config"), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		Expect(err).To(BeNil(), "Should not error trying to open config file")
		f.WriteString(fmt.Sprintf(`[remote "origin"]
    url = %v
    fetch = +refs/heads/*:refs/remotes/origin/*
    git-lob-path = %v
    git-lob-provider = filesystem
[remote "fork"]
    url = %v
    fetch = +refs/heads/*:refs/remotes/fork/*
    git-lob-path = %v
    git-lob-provider = filesystem
`, originPathUrl, originBinStore, forkPathUrl, forkBinStore))
		f.Close()

		LoadConfig(GlobalOptions)
		InitCoreProviders()

		// Create 3 commits with binary file references on master
		mastershaspercommit = CreateManyCommitsForTest(masterfilespercommit, 0, sizeForFile)
		// create another branch, from Tag1 which previous call created on 2nd commit
		err = exec.Command("git", "checkout", "-b", "branch2", "Tag1").Run()
		Expect(err).To(BeNil(), "Didn't create branch")
		// Create 2 more commits on this branch
		branch2shaspercommit = CreateManyCommitsForTest(branch2filespercommit, 3, sizeForFile)
		// Go back
		err = exec.Command("git", "checkout", "master").Run()
		Expect(err).To(BeNil(), "Didn't checkout master")
		// Note that working copy won't have correct binary data because filters aren't necessarily configured
		// see integration_test.go for those tests

	})
	AfterEach(func() {
		os.Chdir(oldwd)
		os.RemoveAll(root)
		os.RemoveAll(originRoot)
		os.RemoveAll(forkRoot)
		// Reset any option changes
		GlobalOptions = NewOptions()
	})

	It("Pushes correctly (Basic)", func() {
		originprovider, err := GetProviderForRemote("origin")
		Expect(err).To(BeNil(), "Shouldn't be an issue getting provider")

		var filesTransferred int
		var filesSkipped int
		var filesFailed int
		var commitsNotFound int
		callback := func(data *ProgressCallbackData) (abort bool) {
			switch data.Type {
			case ProgressTransferBytes:
				if data.ItemBytesDone == data.ItemBytes {
					filesTransferred++
				}
			case ProgressSkip:
				filesSkipped++
			case ProgressError:
				filesFailed++
			case ProgressNotFound:
				commitsNotFound++
			}
			return false
		}
		Expect(HasPushedBinaryState("origin")).To(BeFalse(), "Should not have pushed state for origin")
		Expect(HasPushedBinaryState("fork")).To(BeFalse(), "Should not have pushed state for fork")

		// Start by pushing up to Tag1 so that we push only first 2 commits on master
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "Tag1"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount := (len(masterfilespercommit[0]) + len(masterfilespercommit[1])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped")
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(0), "No files should be not found")

		// Should now have pushed state
		Expect(HasPushedBinaryState("origin")).To(BeTrue(), "Should have pushed state for origin")
		// Check it's at position expected
		mastersha, _ := GitRefToFullSHA("master")
		pushedSHA, err := FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		tag1sha, _ := GitRefToFullSHA("Tag1")
		Expect(pushedSHA).To(Equal(tag1sha), "Pushed marker should be at Tag1")

		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[0], originBinStore)
		checkLOBsExist(mastershaspercommit[1], originBinStore)

		// Now push all of master, should skip previous & upload new
		filesTransferred = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = len(masterfilespercommit[2]) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(0), "No files should be not found")
		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[2], originBinStore)

		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(mastersha), "Pushed marker should be at master")

		// Now push a different branch
		// Now push all of master, should skip previous & upload new
		filesTransferred = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "branch2"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = (len(branch2filespercommit[0]) + len(branch2filespercommit[1])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(0), "No files should be not found")
		// Confirm data exists on remote
		checkLOBsExist(branch2shaspercommit[0], originBinStore)
		checkLOBsExist(branch2shaspercommit[1], originBinStore)

		branch2sha, _ := GitRefToFullSHA("branch2")
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", branch2sha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(branch2sha), "Pushed marker should be at branch2")

		// Now push master to fork
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("fork", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(""), "Pushed marker should not be set for fork")

		filesTransferred = 0
		err = PushBasic(originprovider, "fork", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		expectedFileCount = (len(masterfilespercommit[0]) + len(masterfilespercommit[1]) + len(masterfilespercommit[2])) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(0), "No files should be skipped") // because cache should prevent
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(0), "No files should be not found")
		// Confirm data exists on remote
		checkLOBsExist(mastershaspercommit[0], forkBinStore)
		checkLOBsExist(mastershaspercommit[1], forkBinStore)
		Expect(HasPushedBinaryState("fork")).To(BeTrue(), "Should have pushed state for fork")
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("fork", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(mastersha), "Pushed marker should be at master")

		// now reset all the pushed data for origin
		err = ResetPushedBinaryState("origin")
		Expect(err).To(BeNil(), "Should not be error resetting pushed data")
		Expect(HasPushedBinaryState("origin")).To(BeFalse(), "Should not have pushed state for origin")
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(""), "Pushed marker should not be set")

		// now delete some of the local LOBs to create a gap in our data
		// but leave the data on the remote; this simulates the case where user has fetched commits from
		// someone else but hasn't fetched LOBs, then tries to push their own binaries
		// this should still succeed, and because LOBs are on the remote then it's fine to
		// move the pushed pointer over these commits
		removeLOBs(mastershaspercommit[1], GetLocalLOBRoot())
		// Also delete some *other* LOBs on the remote to make sure they get pushed
		removeLOBs(mastershaspercommit[2], originBinStore)

		// now push master again, should be OK to skip over missing LOBs since on remote
		filesTransferred = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		// We should transfer [2] because not on remote
		// We should do nothing with [1] - missing locally but OK because on remote
		// And should skip [0] since already on remote and local
		expectedFileCount = len(masterfilespercommit[2]) * 2
		expectedSkipFileCount := len(masterfilespercommit[0]) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(expectedSkipFileCount), "Should skip the files already on remote")
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(0), "Should have no 'not found' files since found on remote")
		// Confirm new data exists on remote
		checkLOBsExist(mastershaspercommit[2], originBinStore)
		// Check that push cache has been updated (because missing files were OK on remote)
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(mastersha), "Pushed marker should be at master")

		// Reset all the state again, and this time we'll test with missing data locally that's
		// also missing on the remote
		err = ResetPushedBinaryState("origin")
		Expect(err).To(BeNil(), "Should not be error resetting pushed data")
		Expect(HasPushedBinaryState("origin")).To(BeFalse(), "Should not have pushed state for origin")
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(""), "Pushed marker should not be set")

		// now delete some of the local AND remote LOBs to create a gap in our data
		// We should still push data we have but not update push cache state, should warn about missing
		removeLOBs(mastershaspercommit[1], GetLocalLOBRoot())
		removeLOBs(mastershaspercommit[1], originBinStore)
		// Also delete some *other* LOBs on the remote to make sure they get pushed
		removeLOBs(mastershaspercommit[2], originBinStore)

		// now push master again, should be OK to skip over missing LOBs since on remote
		filesTransferred = 0
		filesSkipped = 0
		err = PushBasic(originprovider, "origin", []*GitRefSpec{&GitRefSpec{Ref1: "master"}}, false, false, false, callback)
		Expect(err).To(BeNil(), "Push should succeed")
		// Files should equal 2 for each entry (meta + one chunk)
		// We should transfer [2] because not on remote & present locally
		// We should 'not found' on [1]
		// And should skip [0] since already on remote and local
		expectedFileCount = len(masterfilespercommit[2]) * 2
		expectedSkipFileCount = len(masterfilespercommit[0]) * 2
		Expect(filesTransferred).To(BeEquivalentTo(expectedFileCount), "Should have transferred the right number of files")
		Expect(filesSkipped).To(BeEquivalentTo(expectedSkipFileCount), "Should skip the files already on remote")
		Expect(filesFailed).To(BeEquivalentTo(0), "No files should fail")
		Expect(commitsNotFound).To(BeEquivalentTo(1), "One commit should have had missing files locally & on remote")
		// Confirm new data exists on remote
		checkLOBsExist(mastershaspercommit[2], originBinStore)
		// Check that push cache has been updated, but only to [0] (Tag0)
		tag0sha, _ := GitRefToFullSHA("Tag0")
		pushedSHA, err = FindLatestAncestorWhereBinariesPushed("origin", mastersha)
		Expect(err).To(BeNil(), "Should not be error finding latest pushed")
		Expect(pushedSHA).To(Equal(tag0sha), "Pushed marker should have only been moved to the point before missing files on local & remote")

	})

})
<|endoftext|>


<commit_before>// Import modules

package py

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	// This will become sys.path one day ;-)
	modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"}
)

// The workings of __import__
//
// __import__(name, globals=None, locals=None, fromlist=(), level=0)
//
// This function is invoked by the import statement. It can be
// replaced (by importing the builtins module and assigning to
// builtins.__import__) in order to change semantics of the import
// statement, but doing so is strongly discouraged as it is usually
// simpler to use import hooks (see PEP 302) to attain the same goals
// and does not cause issues with code which assumes the default
// import implementation is in use. Direct use of __import__() is also
// discouraged in favor of importlib.import_module().
//
// The function imports the module name, potentially using the given
// globals and locals to determine how to interpret the name in a
// package context. The fromlist gives the names of objects or
// submodules that should be imported from the module given by
// name. The standard implementation does not use its locals argument
// at all, and uses its globals only to determine the package context
// of the import statement.
//
// level specifies whether to use absolute or relative imports. 0 (the
// default) means only perform absolute imports. Positive values for
// level indicate the number of parent directories to search relative
// to the directory of the module calling __import__() (see PEP 328
// for the details).
//
// When the name variable is of the form package.module, normally, the
// top-level package (the name up till the first dot) is returned, not
// the module named by name. However, when a non-empty fromlist
// argument is given, the module named by name is returned.
//
// For example, the statement import spam results in bytecode
// resembling the following code:
//
// spam = __import__('spam', globals(), locals(), [], 0)
// The statement import spam.ham results in this call:
//
// spam = __import__('spam.ham', globals(), locals(), [], 0)
//
// Note how __import__() returns the toplevel module here because this
// is the object that is bound to a name by the import statement.
//
// On the other hand, the statement from spam.ham import eggs, sausage
// as saus results in
//
// _temp = __import__('spam.ham', globals(), locals(), ['eggs', 'sausage'], 0)
// eggs = _temp.eggs
// saus = _temp.sausage
//
// Here, the spam.ham module is returned from __import__(). From this
// object, the names to import are retrieved and assigned to their
// respective names.
//
// If you simply want to import a module (potentially within a
// package) by name, use importlib.import_module().
//
// Changed in version 3.3: Negative values for level are no longer
// supported (which also changes the default value to 0).
func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) Object {
	// Module already loaded - return that
	if module, ok := modules[name]; ok {
		return module
	}

	if level != 0 {
		panic("Relative import not supported yet")
	}

	parts := strings.Split(name, ".")
	pathParts := path.Join(parts...)

	for _, mpath := range modulePath {
		if mpath == "" {
			mpathObj, ok := globals["__file__"]
			if !ok {
				panic(ExceptionNewf(SystemError, "Couldn't find __file__ in globals"))
			}
			mpath = path.Dir(string(mpathObj.(String)))
		}
		fullPath := path.Join(mpath, pathParts)
		// FIXME Read pyc/pyo too
		fullPath, err := filepath.Abs(fullPath + ".py")
		if err != nil {
			continue
		}
		// Check if file exists
		if _, err := os.Stat(fullPath); err == nil {
			str, err := ioutil.ReadFile(fullPath)
			if err != nil {
				panic(ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err))
			}
			codeObj, err := Compile(string(str), fullPath, "exec", 0, true)
			if err != nil {
				panic(err) // FIXME error handling
			}
			code, ok := codeObj.(*Code)
			if !ok {
				panic(ExceptionNewf(ImportError, "Compile didn't return code object"))
			}
			module := NewModule(name, "", nil, nil)
			_, err = VmRun(module.Globals, module.Globals, code, nil)
			if err != nil {
				panic(err)
			}
			module.Globals["__file__"] = String(fullPath)
			return module
		}
	}
	panic(ExceptionNewf(ImportError, "No module named '%s'", name))

	// Convert to absolute path if relative
	// Use __file__ from globals to work out what we are relative to

	// '' in path seems to mean use the current __file__

	// Find a valid path which we need to check for the correct __init__.py in subdirectories etc

	// Look for .py and .pyc files

	// Make absolute module path too if we can for sys.modules

	//How do we uniquely identify modules?

	// SystemError: Parent module '' not loaded, cannot perform relative import

}

// Straight port of the python code
//
// This calls functins from _bootstrap.py which is a frozen module
//
// Too much functionality for the moment
func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) Object {
	var abs_name string
	var builtins_import Object
	var final_mod Object
	var mod Object
	var PackageObj Object
	var Package string
	var globals StringDict
	var fromlist Tuple
	var ok bool
	var name string

	// Make sure to use default values so as to not have
	// PyObject_CallMethodObjArgs() truncate the parameter list because of a
	// nil argument.
	if given_globals == nil {
		globals = StringDict{}
	} else {
		// Only have to care what given_globals is if it will be used
		// for something.
		globals, ok = given_globals.(StringDict)
		if level > 0 && !ok {
			panic(ExceptionNewf(TypeError, "globals must be a dict"))
		}
	}

	if given_fromlist == nil || given_fromlist == None {
		fromlist = Tuple{}
	} else {
		fromlist = SequenceTuple(given_fromlist)
	}
	if nameObj == nil {
		panic(ExceptionNewf(ValueError, "Empty module name"))
	}

	// The below code is importlib.__import__() & _gcd_import(), ported to Go
	// for added performance.

	_, ok = nameObj.(String)
	if !ok {
		panic(ExceptionNewf(TypeError, "module name must be a string"))
	}
	name = string(nameObj.(String))

	if level < 0 {
		panic(ExceptionNewf(ValueError, "level must be >= 0"))
	} else if level > 0 {
		PackageObj, ok = globals["__package__"]
		if ok && PackageObj != None {
			if _, ok = PackageObj.(String); !ok {
				panic(ExceptionNewf(TypeError, "package must be a string"))
			}
			Package = string(PackageObj.(String))
		} else {
			PackageObj, ok = globals["__name__"]
			if !ok {
				panic(ExceptionNewf(KeyError, "'__name__' not in globals"))
			} else if _, ok = PackageObj.(String); !ok {
				panic(ExceptionNewf(TypeError, "__name__ must be a string"))
			}
			Package = string(PackageObj.(String))

			if _, ok = globals["__path__"]; !ok {
				i := strings.LastIndex(string(Package), ".")
				if i < 0 {
					Package = ""
				} else {
					Package = Package[:i]
				}
			}
		}

		if _, ok = modules[string(Package)]; !ok {
			panic(ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package))
		}
	} else { // level == 0 */
		if len(name) == 0 {
			panic(ExceptionNewf(ValueError, "Empty module name"))
		}
		Package = ""
	}

	if level > 0 {
		last_dot := len(Package)
		var base string
		level_up := 1

		for level_up = 1; level_up < level; level_up += 1 {
			last_dot = strings.LastIndex(string(Package[:last_dot]), ".")
			if last_dot < 0 {
				panic(ExceptionNewf(ValueError, "attempted relative import beyond top-level Package"))
			}
		}

		base = Package[:last_dot]

		if len(name) > 0 {
			abs_name = strings.Join([]string{base, name}, ".")
		} else {
			abs_name = base
		}
	} else {
		abs_name = name
	}

	// FIXME _PyImport_AcquireLock()

	// From this point forward, goto error_with_unlock!
	builtins_import, ok = globals["__import__"]
	if !ok {
		builtins_import, ok = Builtins.Globals["__import__"]
		if !ok {
			panic(ExceptionNewf(ImportError, "__import__ not found"))
		}
	}

	mod, ok = modules[abs_name]
	if mod == None {
		panic(ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name))
	} else if ok {
		var value Object
		var err error
		initializing := false

		// Optimization: only call _bootstrap._lock_unlock_module() if
		// __initializing__ is true.
		// NOTE: because of this, __initializing__ must be set *before*
		// stuffing the new module in sys.modules.

		value, err = GetAttrStringErr(mod, "__initializing__")
		if err == nil {
			initializing = bool(MakeBool(value).(Bool))
		}
		if initializing {
			// _bootstrap._lock_unlock_module() releases the import lock */
			value = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil)
		} else {
			// FIXME locking
			// if _PyImport_ReleaseLock() < 0 {
			// 	panic(ExceptionNewf(RuntimeError, "not holding the import lock"))
			// }
		}
	} else {
		// _bootstrap._find_and_load() releases the import lock
		mod = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil)
	}
	// From now on we don't hold the import lock anymore.

	if len(fromlist) == 0 {
		if level == 0 || len(name) > 0 {
			i := strings.Index(name, ".")
			if i < 0 {
				// No dot in module name, simple exit
				final_mod = mod
				goto error
			}
			front := name[:1]

			if level == 0 {
				final_mod = Call(builtins_import, Tuple{String(front)}, nil)
			} else {
				cut_off := len(name) - len(front)
				abs_name_len := len(abs_name)
				to_return := abs_name[:abs_name_len-cut_off]
				final_mod, ok = modules[to_return]
				if !ok {
					panic(ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return))
				}
			}
		} else {
			final_mod = mod
		}
	} else {
		final_mod = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil)
	}
	goto error

	//error_with_unlock:
	// FIXME defer?
	// if _PyImport_ReleaseLock() < 0 {
	// 	panic(ExceptionNewf(RuntimeError, "not holding the import lock")
	// }
error:
	// FIXME defer?
	// if final_mod == nil {
	// 	remove_importlib_frames()
	// }
	return final_mod
}
<commit_msg>py: import - make work better in REPL and look for __init__.py<commit_after>// Import modules

package py

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	// This will become sys.path one day ;-)
	modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"}
)

// The workings of __import__
//
// __import__(name, globals=None, locals=None, fromlist=(), level=0)
//
// This function is invoked by the import statement. It can be
// replaced (by importing the builtins module and assigning to
// builtins.__import__) in order to change semantics of the import
// statement, but doing so is strongly discouraged as it is usually
// simpler to use import hooks (see PEP 302) to attain the same goals
// and does not cause issues with code which assumes the default
// import implementation is in use. Direct use of __import__() is also
// discouraged in favor of importlib.import_module().
//
// The function imports the module name, potentially using the given
// globals and locals to determine how to interpret the name in a
// package context. The fromlist gives the names of objects or
// submodules that should be imported from the module given by
// name. The standard implementation does not use its locals argument
// at all, and uses its globals only to determine the package context
// of the import statement.
//
// level specifies whether to use absolute or relative imports. 0 (the
// default) means only perform absolute imports. Positive values for
// level indicate the number of parent directories to search relative
// to the directory of the module calling __import__() (see PEP 328
// for the details).
//
// When the name variable is of the form package.module, normally, the
// top-level package (the name up till the first dot) is returned, not
// the module named by name. However, when a non-empty fromlist
// argument is given, the module named by name is returned.
//
// For example, the statement import spam results in bytecode
// resembling the following code:
//
// spam = __import__('spam', globals(), locals(), [], 0)
// The statement import spam.ham results in this call:
//
// spam = __import__('spam.ham', globals(), locals(), [], 0)
//
// Note how __import__() returns the toplevel module here because this
// is the object that is bound to a name by the import statement.
//
// On the other hand, the statement from spam.ham import eggs, sausage
// as saus results in
//
// _temp = __import__('spam.ham', globals(), locals(), ['eggs', 'sausage'], 0)
// eggs = _temp.eggs
// saus = _temp.sausage
//
// Here, the spam.ham module is returned from __import__(). From this
// object, the names to import are retrieved and assigned to their
// respective names.
//
// If you simply want to import a module (potentially within a
// package) by name, use importlib.import_module().
//
// Changed in version 3.3: Negative values for level are no longer
// supported (which also changes the default value to 0).
func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) Object {
	// Module already loaded - return that
	if module, ok := modules[name]; ok {
		return module
	}

	if level != 0 {
		panic("Relative import not supported yet")
	}

	parts := strings.Split(name, ".")
	pathParts := path.Join(parts...)

	for _, mpath := range modulePath {
		if mpath == "" {
			mpathObj, ok := globals["__file__"]
			if !ok {
				var err error
				mpath, err = os.Getwd()
				if err != nil {
					panic(err)
				}
			} else {
				mpath = path.Dir(string(mpathObj.(String)))
			}
		}
		fullPath := path.Join(mpath, pathParts)
		// FIXME Read pyc/pyo too
		fullPath, err := filepath.Abs(fullPath)
		if err != nil {
			continue
		}
		if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() {
			// FIXME this is a massive simplification!
			fullPath = path.Join(fullPath, "__init__.py")
		} else {
			fullPath += ".py"
		}
		// Check if file exists
		if _, err := os.Stat(fullPath); err == nil {
			str, err := ioutil.ReadFile(fullPath)
			if err != nil {
				panic(ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err))
			}
			codeObj, err := Compile(string(str), fullPath, "exec", 0, true)
			if err != nil {
				panic(err) // FIXME error handling
			}
			code, ok := codeObj.(*Code)
			if !ok {
				panic(ExceptionNewf(ImportError, "Compile didn't return code object"))
			}
			module := NewModule(name, "", nil, nil)
			_, err = VmRun(module.Globals, module.Globals, code, nil)
			if err != nil {
				panic(err)
			}
			module.Globals["__file__"] = String(fullPath)
			return module
		}
	}
	panic(ExceptionNewf(ImportError, "No module named '%s'", name))

	// Convert to absolute path if relative
	// Use __file__ from globals to work out what we are relative to

	// '' in path seems to mean use the current __file__

	// Find a valid path which we need to check for the correct __init__.py in subdirectories etc

	// Look for .py and .pyc files

	// Make absolute module path too if we can for sys.modules

	//How do we uniquely identify modules?

	// SystemError: Parent module '' not loaded, cannot perform relative import

}

// Straight port of the python code
//
// This calls functins from _bootstrap.py which is a frozen module
//
// Too much functionality for the moment
func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) Object {
	var abs_name string
	var builtins_import Object
	var final_mod Object
	var mod Object
	var PackageObj Object
	var Package string
	var globals StringDict
	var fromlist Tuple
	var ok bool
	var name string

	// Make sure to use default values so as to not have
	// PyObject_CallMethodObjArgs() truncate the parameter list because of a
	// nil argument.
	if given_globals == nil {
		globals = StringDict{}
	} else {
		// Only have to care what given_globals is if it will be used
		// for something.
		globals, ok = given_globals.(StringDict)
		if level > 0 && !ok {
			panic(ExceptionNewf(TypeError, "globals must be a dict"))
		}
	}

	if given_fromlist == nil || given_fromlist == None {
		fromlist = Tuple{}
	} else {
		fromlist = SequenceTuple(given_fromlist)
	}
	if nameObj == nil {
		panic(ExceptionNewf(ValueError, "Empty module name"))
	}

	// The below code is importlib.__import__() & _gcd_import(), ported to Go
	// for added performance.

	_, ok = nameObj.(String)
	if !ok {
		panic(ExceptionNewf(TypeError, "module name must be a string"))
	}
	name = string(nameObj.(String))

	if level < 0 {
		panic(ExceptionNewf(ValueError, "level must be >= 0"))
	} else if level > 0 {
		PackageObj, ok = globals["__package__"]
		if ok && PackageObj != None {
			if _, ok = PackageObj.(String); !ok {
				panic(ExceptionNewf(TypeError, "package must be a string"))
			}
			Package = string(PackageObj.(String))
		} else {
			PackageObj, ok = globals["__name__"]
			if !ok {
				panic(ExceptionNewf(KeyError, "'__name__' not in globals"))
			} else if _, ok = PackageObj.(String); !ok {
				panic(ExceptionNewf(TypeError, "__name__ must be a string"))
			}
			Package = string(PackageObj.(String))

			if _, ok = globals["__path__"]; !ok {
				i := strings.LastIndex(string(Package), ".")
				if i < 0 {
					Package = ""
				} else {
					Package = Package[:i]
				}
			}
		}

		if _, ok = modules[string(Package)]; !ok {
			panic(ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package))
		}
	} else { // level == 0 */
		if len(name) == 0 {
			panic(ExceptionNewf(ValueError, "Empty module name"))
		}
		Package = ""
	}

	if level > 0 {
		last_dot := len(Package)
		var base string
		level_up := 1

		for level_up = 1; level_up < level; level_up += 1 {
			last_dot = strings.LastIndex(string(Package[:last_dot]), ".")
			if last_dot < 0 {
				panic(ExceptionNewf(ValueError, "attempted relative import beyond top-level Package"))
			}
		}

		base = Package[:last_dot]

		if len(name) > 0 {
			abs_name = strings.Join([]string{base, name}, ".")
		} else {
			abs_name = base
		}
	} else {
		abs_name = name
	}

	// FIXME _PyImport_AcquireLock()

	// From this point forward, goto error_with_unlock!
	builtins_import, ok = globals["__import__"]
	if !ok {
		builtins_import, ok = Builtins.Globals["__import__"]
		if !ok {
			panic(ExceptionNewf(ImportError, "__import__ not found"))
		}
	}

	mod, ok = modules[abs_name]
	if mod == None {
		panic(ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name))
	} else if ok {
		var value Object
		var err error
		initializing := false

		// Optimization: only call _bootstrap._lock_unlock_module() if
		// __initializing__ is true.
		// NOTE: because of this, __initializing__ must be set *before*
		// stuffing the new module in sys.modules.

		value, err = GetAttrStringErr(mod, "__initializing__")
		if err == nil {
			initializing = bool(MakeBool(value).(Bool))
		}
		if initializing {
			// _bootstrap._lock_unlock_module() releases the import lock */
			value = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil)
		} else {
			// FIXME locking
			// if _PyImport_ReleaseLock() < 0 {
			// 	panic(ExceptionNewf(RuntimeError, "not holding the import lock"))
			// }
		}
	} else {
		// _bootstrap._find_and_load() releases the import lock
		mod = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil)
	}
	// From now on we don't hold the import lock anymore.

	if len(fromlist) == 0 {
		if level == 0 || len(name) > 0 {
			i := strings.Index(name, ".")
			if i < 0 {
				// No dot in module name, simple exit
				final_mod = mod
				goto error
			}
			front := name[:1]

			if level == 0 {
				final_mod = Call(builtins_import, Tuple{String(front)}, nil)
			} else {
				cut_off := len(name) - len(front)
				abs_name_len := len(abs_name)
				to_return := abs_name[:abs_name_len-cut_off]
				final_mod, ok = modules[to_return]
				if !ok {
					panic(ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return))
				}
			}
		} else {
			final_mod = mod
		}
	} else {
		final_mod = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil)
	}
	goto error

	//error_with_unlock:
	// FIXME defer?
	// if _PyImport_ReleaseLock() < 0 {
	// 	panic(ExceptionNewf(RuntimeError, "not holding the import lock")
	// }
error:
	// FIXME defer?
	// if final_mod == nil {
	// 	remove_importlib_frames()
	// }
	return final_mod
}
<|endoftext|>


<commit_before>package crawler

import (
	"sync"
	"time"

	"github.com/pkg/errors"
	"github.com/utahta/go-atomicbool"
	"github.com/utahta/momoclo-channel/lib/log"
	"github.com/utahta/momoclo-channel/model"
	"github.com/utahta/momoclo-crawler"
	"golang.org/x/net/context"
	"google.golang.org/appengine/urlfetch"
)

var timeNow = time.Now

func Crawl(ctx context.Context) error {
	ctx, cancel := context.WithTimeout(ctx, 50*time.Second)
	defer cancel()

	var workQueue = make(chan bool, 20)
	defer close(workQueue)

	clients := crawlChannelClients(ctx)

	errFlg := atomicbool.New(false)
	var wg sync.WaitGroup
	wg.Add(len(clients))
	for _, cli := range clients {
		workQueue <- true
		go func(ctx context.Context, cli *crawler.ChannelClient) {
			defer func() {
				<-workQueue
				wg.Done()
			}()

			ch, err := cli.Fetch()
			if err != nil {
				errFlg.Set(true)
				log.Error(ctx, err)
				return
			}

			// update latest entry
			for _, item := range ch.Items {
				if _, err := model.PutLatestEntry(ctx, item.Url); err != nil {
					log.Error(ctx, err)
					// go on
				}
			}

			q := NewQueueTask()
			if err := q.PushTweet(ctx, ch); err != nil {
				errFlg.Set(true)
				log.Error(ctx, err)
			}
			if err := q.PushLine(ctx, ch); err != nil {
				errFlg.Set(true)
				log.Error(ctx, err)
			}
		}(ctx, cli)
	}
	wg.Wait()

	if errFlg.Enabled() {
		return errors.New("Errors occurred in crawler.Crawl.")
	}
	return nil
}

func crawlChannelClients(ctx context.Context) []*crawler.ChannelClient {
	option := crawler.WithHTTPClient(urlfetch.Client(ctx))
	clients := []*crawler.ChannelClient{
		retrieveChannelClient(crawler.NewTamaiBlogChannelClient(1, model.GetTamaiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewMomotaBlogChannelClient(1, model.GetMomotaLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewAriyasuBlogChannelClient(1, model.GetAriyasuLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewSasakiBlogChannelClient(1, model.GetSasakiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewTakagiBlogChannelClient(1, model.GetTakagiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewAeNewsChannelClient(option)),
		retrieveChannelClient(crawler.NewYoutubeChannelClient(option)),
	}
	jst := time.FixedZone("Asia/Tokyo", 9*60*60)
	now := timeNow().In(jst)

	// every week on Sunday, 16:55 <= now <= 17:59 || 20:00 <= now <= 20:59
	if now.Weekday() == time.Sunday && ((now.Hour() == 16 && now.Minute() >= 55) || now.Hour() == 17 || now.Hour() == 20) {
		clients = append(clients, retrieveChannelClient(crawler.NewHappycloChannelClient(model.GetHappycloLatestEntryURL(ctx), option)))
	}

	return clients
}

func retrieveChannelClient(c *crawler.ChannelClient, _ error) *crawler.ChannelClient {
	return c
}
<commit_msg>Use errgroup<commit_after>package crawler

import (
	"time"

	"github.com/pkg/errors"
	"github.com/utahta/momoclo-channel/lib/log"
	"github.com/utahta/momoclo-channel/model"
	"github.com/utahta/momoclo-crawler"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
	"google.golang.org/appengine/urlfetch"
)

var timeNow = time.Now

func Crawl(ctx context.Context) error {
	ctx, cancel := context.WithTimeout(ctx, 50*time.Second)
	defer cancel()

	var workQueue = make(chan bool, 20)
	defer close(workQueue)

	clients := crawlChannelClients(ctx)
	eg := &errgroup.Group{}
	for _, cli := range clients {
		workQueue <- true
		cli := cli
		eg.Go(func() error {
			defer func() {
				<-workQueue
			}()

			ch, err := cli.Fetch()
			if err != nil {
				log.Error(ctx, err)
				return err
			}

			// update latest entry
			for _, item := range ch.Items {
				if _, err := model.PutLatestEntry(ctx, item.Url); err != nil {
					log.Errorf(ctx, "Failed to put latest entry. err:%v", err)
					// go on
				}
			}

			q := NewQueueTask()
			if err := q.PushTweet(ctx, ch); err != nil {
				log.Errorf(ctx, "Failed to push tweet queue. err:%v", err)
			}
			if err := q.PushLine(ctx, ch); err != nil {
				log.Errorf(ctx, "Failed to push line queue. err:%v", err)
			}
			return nil
		})
	}

	if err := eg.Wait(); err != nil {
		return errors.Errorf("Errors occurred in crawler.Crawl. err:%v", err)
	}
	return nil
}

func crawlChannelClients(ctx context.Context) []*crawler.ChannelClient {
	option := crawler.WithHTTPClient(urlfetch.Client(ctx))
	clients := []*crawler.ChannelClient{
		retrieveChannelClient(crawler.NewTamaiBlogChannelClient(1, model.GetTamaiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewMomotaBlogChannelClient(1, model.GetMomotaLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewAriyasuBlogChannelClient(1, model.GetAriyasuLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewSasakiBlogChannelClient(1, model.GetSasakiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewTakagiBlogChannelClient(1, model.GetTakagiLatestEntryURL(ctx), option)),
		retrieveChannelClient(crawler.NewAeNewsChannelClient(option)),
		retrieveChannelClient(crawler.NewYoutubeChannelClient(option)),
	}
	jst := time.FixedZone("Asia/Tokyo", 9*60*60)
	now := timeNow().In(jst)

	// every week on Sunday, 16:55 <= now <= 17:59 || 20:00 <= now <= 20:59
	if now.Weekday() == time.Sunday && ((now.Hour() == 16 && now.Minute() >= 55) || now.Hour() == 17 || now.Hour() == 20) {
		clients = append(clients, retrieveChannelClient(crawler.NewHappycloChannelClient(model.GetHappycloLatestEntryURL(ctx), option)))
	}

	return clients
}

func retrieveChannelClient(c *crawler.ChannelClient, _ error) *crawler.ChannelClient {
	return c
}
<|endoftext|>


<commit_before>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package dialer

import (
	"context"
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/syncthing/syncthing/lib/connections/registry"
	"golang.org/x/net/ipv4"
	"golang.org/x/net/ipv6"
	"golang.org/x/net/proxy"
)

var errUnexpectedInterfaceType = errors.New("unexpected interface type")

// SetTCPOptions sets our default TCP options on a TCP connection, possibly
// digging through dialerConn to extract the *net.TCPConn
func SetTCPOptions(conn net.Conn) error {
	switch conn := conn.(type) {
	case dialerConn:
		return SetTCPOptions(conn.Conn)
	case *net.TCPConn:
		var err error
		if err = conn.SetLinger(0); err != nil {
			return err
		}
		if err = conn.SetNoDelay(false); err != nil {
			return err
		}
		if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
			return err
		}
		if err = conn.SetKeepAlive(true); err != nil {
			return err
		}
		return nil
	default:
		return fmt.Errorf("unknown connection type %T", conn)
	}
}

func SetTrafficClass(conn net.Conn, class int) error {
	switch conn := conn.(type) {
	case dialerConn:
		return SetTrafficClass(conn.Conn, class)
	case *net.TCPConn:
		e1 := ipv4.NewConn(conn).SetTOS(class)
		e2 := ipv6.NewConn(conn).SetTrafficClass(class)

		if e1 != nil {
			return e1
		}
		return e2
	default:
		return fmt.Errorf("unknown connection type %T", conn)
	}
}

func dialContextWithFallback(ctx context.Context, fallback proxy.ContextDialer, network, addr string) (net.Conn, error) {
	dialer, ok := proxy.FromEnvironment().(proxy.ContextDialer)
	if !ok {
		return nil, errUnexpectedInterfaceType
	}
	if dialer == proxy.Direct {
		conn, err := fallback.DialContext(ctx, network, addr)
		l.Debugf("Dialing direct result %s %s: %v %v", network, addr, conn, err)
		return conn, err
	}
	if noFallback {
		conn, err := dialer.DialContext(ctx, network, addr)
		l.Debugf("Dialing no fallback result %s %s: %v %v", network, addr, conn, err)
		if err != nil {
			return nil, err
		}
		return dialerConn{conn, newDialerAddr(network, addr)}, nil
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	var proxyConn, fallbackConn net.Conn
	var proxyErr, fallbackErr error
	proxyDone := make(chan struct{})
	fallbackDone := make(chan struct{})
	go func() {
		proxyConn, proxyErr = dialer.DialContext(ctx, network, addr)
		l.Debugf("Dialing proxy result %s %s: %v %v", network, addr, proxyConn, proxyErr)
		if proxyErr == nil {
			proxyConn = dialerConn{proxyConn, newDialerAddr(network, addr)}
		}
		close(proxyDone)
	}()
	go func() {
		fallbackConn, fallbackErr = fallback.DialContext(ctx, network, addr)
		l.Debugf("Dialing fallback result %s %s: %v %v", network, addr, fallbackConn, fallbackErr)
		close(fallbackDone)
	}()
	<-proxyDone
	if proxyErr == nil {
		go func() {
			<-fallbackDone
			if fallbackErr == nil {
				_ = fallbackConn.Close()
			}
		}()
		return proxyConn, nil
	}
	<-fallbackDone
	return fallbackConn, fallbackErr
}

// DialContext dials via context and/or directly, depending on how it is configured.
// If dialing via proxy and allowing fallback, dialing for both happens simultaneously
// and the proxy connection is returned if successful.
func DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
	return dialContextWithFallback(ctx, proxy.Direct, network, addr)
}

// DialContextReusePort tries dialing via proxy if a proxy is configured, and falls back to
// a direct connection reusing the port from the connections registry, if no proxy is defined, or connecting via proxy
// fails. If the context has a timeout, the timeout might be applied twice.
func DialContextReusePort(ctx context.Context, network, addr string) (net.Conn, error) {
	dialer := &net.Dialer{
		Control: ReusePortControl,
	}
	localAddrInterface := registry.Get(network, tcpAddrLess)
	if localAddrInterface != nil {
		if addr, ok := localAddrInterface.(*net.TCPAddr); !ok {
			return nil, errUnexpectedInterfaceType
		} else {
			dialer.LocalAddr = addr
		}
	}

	return dialContextWithFallback(ctx, dialer, network, addr)
}
<commit_msg>lib/dialer: Try dialing without reuse in parallel (fixes #6892) (#6893)<commit_after>// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package dialer

import (
	"context"
	"errors"
	"fmt"
	"net"
	"time"

	"github.com/syncthing/syncthing/lib/connections/registry"
	"golang.org/x/net/ipv4"
	"golang.org/x/net/ipv6"
	"golang.org/x/net/proxy"
)

var errUnexpectedInterfaceType = errors.New("unexpected interface type")

// SetTCPOptions sets our default TCP options on a TCP connection, possibly
// digging through dialerConn to extract the *net.TCPConn
func SetTCPOptions(conn net.Conn) error {
	switch conn := conn.(type) {
	case dialerConn:
		return SetTCPOptions(conn.Conn)
	case *net.TCPConn:
		var err error
		if err = conn.SetLinger(0); err != nil {
			return err
		}
		if err = conn.SetNoDelay(false); err != nil {
			return err
		}
		if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
			return err
		}
		if err = conn.SetKeepAlive(true); err != nil {
			return err
		}
		return nil
	default:
		return fmt.Errorf("unknown connection type %T", conn)
	}
}

func SetTrafficClass(conn net.Conn, class int) error {
	switch conn := conn.(type) {
	case dialerConn:
		return SetTrafficClass(conn.Conn, class)
	case *net.TCPConn:
		e1 := ipv4.NewConn(conn).SetTOS(class)
		e2 := ipv6.NewConn(conn).SetTrafficClass(class)

		if e1 != nil {
			return e1
		}
		return e2
	default:
		return fmt.Errorf("unknown connection type %T", conn)
	}
}

func dialContextWithFallback(ctx context.Context, fallback proxy.ContextDialer, network, addr string) (net.Conn, error) {
	dialer, ok := proxy.FromEnvironment().(proxy.ContextDialer)
	if !ok {
		return nil, errUnexpectedInterfaceType
	}
	if dialer == proxy.Direct {
		conn, err := fallback.DialContext(ctx, network, addr)
		l.Debugf("Dialing direct result %s %s: %v %v", network, addr, conn, err)
		return conn, err
	}
	if noFallback {
		conn, err := dialer.DialContext(ctx, network, addr)
		l.Debugf("Dialing no fallback result %s %s: %v %v", network, addr, conn, err)
		if err != nil {
			return nil, err
		}
		return dialerConn{conn, newDialerAddr(network, addr)}, nil
	}

	proxyDialFudgeAddress := func(ctx context.Context, network, address string) (net.Conn, error) {
		conn, err := dialer.DialContext(ctx, network, addr)
		if err != nil {
			return nil, err
		}
		return dialerConn{conn, newDialerAddr(network, addr)}, err
	}

	return dialTwicePreferFirst(ctx, proxyDialFudgeAddress, fallback.DialContext, "proxy", "fallback", network, addr)
}

// DialContext dials via context and/or directly, depending on how it is configured.
// If dialing via proxy and allowing fallback, dialing for both happens simultaneously
// and the proxy connection is returned if successful.
func DialContext(ctx context.Context, network, addr string) (net.Conn, error) {
	return dialContextWithFallback(ctx, proxy.Direct, network, addr)
}

// DialContextReusePort tries dialing via proxy if a proxy is configured, and falls back to
// a direct connection reusing the port from the connections registry, if no proxy is defined, or connecting via proxy
// fails. It also in parallel dials without reusing the port, just in case reusing the port affects routing decisions badly.
func DialContextReusePort(ctx context.Context, network, addr string) (net.Conn, error) {
	// If proxy is configured, there is no point trying to reuse listen addresses.
	if proxy.FromEnvironment() != proxy.Direct {
		return DialContext(ctx, network, addr)
	}

	localAddrInterface := registry.Get(network, tcpAddrLess)
	if localAddrInterface == nil {
		// Nothing listening, nothing to reuse.
		return DialContext(ctx, network, addr)
	}

	laddr, ok := localAddrInterface.(*net.TCPAddr)
	if !ok {
		return nil, errUnexpectedInterfaceType
	}

	// Dial twice, once reusing the listen address, another time not reusing it, just in case reusing the address
	// influences routing and we fail to reach our destination.
	dialer := net.Dialer{
		Control:   ReusePortControl,
		LocalAddr: laddr,
	}
	return dialTwicePreferFirst(ctx, dialer.DialContext, (&net.Dialer{}).DialContext, "reuse", "non-reuse", network, addr)
}

type dialFunc func(ctx context.Context, network, address string) (net.Conn, error)

func dialTwicePreferFirst(ctx context.Context, first, second dialFunc, firstName, secondName, network, address string) (net.Conn, error) {
	// Delay second dial by some time.
	sleep := time.Second
	if deadline, ok := ctx.Deadline(); ok {
		timeout := time.Until(deadline)
		if timeout > 0 {
			sleep = timeout / 3
		}
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	var firstConn, secondConn net.Conn
	var firstErr, secondErr error
	firstDone := make(chan struct{})
	secondDone := make(chan struct{})
	go func() {
		firstConn, firstErr = first(ctx, network, address)
		l.Debugf("Dialing %s result %s %s: %v %v", firstName, network, address, firstConn, firstErr)
		close(firstDone)
	}()
	go func() {
		select {
		case <-firstDone:
			if firstErr == nil {
				// First succeeded, no point doing anything in second
				secondErr = errors.New("didn't dial")
				close(secondDone)
				return
			}
		case <-ctx.Done():
			secondErr = ctx.Err()
			close(secondDone)
			return
		case <-time.After(sleep):
		}
		secondConn, secondErr = second(ctx, network, address)
		l.Debugf("Dialing %s result %s %s: %v %v", secondName, network, address, secondConn, secondErr)
		close(secondDone)
	}()
	<-firstDone
	if firstErr == nil {
		go func() {
			<-secondDone
			if secondConn != nil {
				_ = secondConn.Close()
			}
		}()
		return firstConn, firstErr
	}
	<-secondDone
	return secondConn, secondErr
}
<|endoftext|>


<commit_before>// Package filestore provide a storage backend based on the local file system.
//
// FileStore is a storage backend used as a tusd.DataStore in tusd.NewHandler.
// It stores the uploads in a directory specified in two different files: The
// `[id].info` files are used to store the fileinfo in JSON format. The
// `[id].bin` files contain the raw binary data uploaded.
// No cleanup is performed so you may want to run a cronjob to ensure your disk
// is not filled up with old and finished uploads.
//
// In addition, it provides an exclusive upload locking mechanism using lock files
// which are stored on disk. Each of them stores the PID of the process which
// acquired the lock. This allows locks to be automatically freed when a process
// is unable to release it on its own because the process is not alive anymore.
// For more information, consult the documentation for tusd.LockerDataStore
// interface, which is implemented by FileStore
package filestore

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/tus/tusd"
	"github.com/tus/tusd/uid"

	"gopkg.in/Acconut/lockfile.v1"
)

var defaultFilePerm = os.FileMode(0664)

// See the tusd.DataStore interface for documentation about the different
// methods.
type FileStore struct {
	// Relative or absolute path to store files in. FileStore does not check
	// whether the path exists, use os.MkdirAll in this case on your own.
	Path string
}

// New creates a new file based storage backend. The directory specified will
// be used as the only storage entry. This method does not check
// whether the path exists, use os.MkdirAll to ensure.
// In addition, a locking mechanism is provided.
func New(path string) FileStore {
	return FileStore{path}
}

// UseIn sets this store as the core data store in the passed composer and adds
// all possible extension to it.
func (store FileStore) UseIn(composer *tusd.StoreComposer) {
	composer.UseCore(store)
	composer.UseGetReader(store)
	composer.UseTerminater(store)
	composer.UseLocker(store)
}

func (store FileStore) NewUpload(info tusd.FileInfo) (id string, err error) {
	id = uid.Uid()
	info.ID = id

	// Create .bin file with no content
	file, err := os.OpenFile(store.binPath(id), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
	if err != nil {
		if os.IsNotExist(err) {
			err = fmt.Errorf("upload directory does not exist: %s", store.Path)
		}
		return "", err
	}
	defer file.Close()

	// writeInfo creates the file by itself if necessary
	err = store.writeInfo(id, info)
	return
}

func (store FileStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
	file, err := os.OpenFile(store.binPath(id), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return 0, err
	}
	defer file.Close()

	n, err := io.Copy(file, src)
	return n, err
}

func (store FileStore) GetInfo(id string) (tusd.FileInfo, error) {
	info := tusd.FileInfo{}
	data, err := ioutil.ReadFile(store.infoPath(id))
	if err != nil {
		return info, err
	}
	if err := json.Unmarshal(data, &info); err != nil {
		return info, err
	}

	stat, err := os.Stat(store.binPath(id))
	if err != nil {
		return info, err
	}

	info.Offset = stat.Size()

	return info, nil
}

func (store FileStore) GetReader(id string) (io.Reader, error) {
	return os.Open(store.binPath(id))
}

func (store FileStore) Terminate(id string) error {
	if err := os.Remove(store.infoPath(id)); err != nil {
		return err
	}
	if err := os.Remove(store.binPath(id)); err != nil {
		return err
	}
	return nil
}

func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) {
	file, err := os.OpenFile(store.binPath(dest), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, id := range uploads {
		src, err := store.GetReader(id)
		if err != nil {
			return err
		}

		if _, err := io.Copy(file, src); err != nil {
			return err
		}
	}

	return
}

func (store FileStore) LockUpload(id string) error {
	lock, err := store.newLock(id)
	if err != nil {
		return err
	}

	err = lock.TryLock()
	if err == lockfile.ErrBusy {
		return tusd.ErrFileLocked
	}

	return err
}

func (store FileStore) UnlockUpload(id string) error {
	lock, err := store.newLock(id)
	if err != nil {
		return err
	}

	err = lock.Unlock()

	// A "no such file or directory" will be returned if no lockfile was found.
	// Since this means that the file has never been locked, we drop the error
	// and continue as if nothing happened.
	if os.IsNotExist(err) {
		err = nil
	}

	return err
}

// newLock contructs a new Lockfile instance.
func (store FileStore) newLock(id string) (lockfile.Lockfile, error) {
	path, err := filepath.Abs(filepath.Join(store.Path, id+".lock"))
	if err != nil {
		return lockfile.Lockfile(""), err
	}

	// We use Lockfile directly instead of lockfile.New to bypass the unnecessary
	// check whether the provided path is absolute since we just resolved it
	// on our own.
	return lockfile.Lockfile(path), nil
}

// binPath returns the path to the .bin storing the binary data.
func (store FileStore) binPath(id string) string {
	return filepath.Join(store.Path, id+".bin")
}

// infoPath returns the path to the .info file storing the file's info.
func (store FileStore) infoPath(id string) string {
	return filepath.Join(store.Path, id+".info")
}

// writeInfo updates the entire information. Everything will be overwritten.
func (store FileStore) writeInfo(id string, info tusd.FileInfo) error {
	data, err := json.Marshal(info)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(store.infoPath(id), data, defaultFilePerm)
}
<commit_msg>Expose concatenation support in filestore<commit_after>// Package filestore provide a storage backend based on the local file system.
//
// FileStore is a storage backend used as a tusd.DataStore in tusd.NewHandler.
// It stores the uploads in a directory specified in two different files: The
// `[id].info` files are used to store the fileinfo in JSON format. The
// `[id].bin` files contain the raw binary data uploaded.
// No cleanup is performed so you may want to run a cronjob to ensure your disk
// is not filled up with old and finished uploads.
//
// In addition, it provides an exclusive upload locking mechanism using lock files
// which are stored on disk. Each of them stores the PID of the process which
// acquired the lock. This allows locks to be automatically freed when a process
// is unable to release it on its own because the process is not alive anymore.
// For more information, consult the documentation for tusd.LockerDataStore
// interface, which is implemented by FileStore
package filestore

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/tus/tusd"
	"github.com/tus/tusd/uid"

	"gopkg.in/Acconut/lockfile.v1"
)

var defaultFilePerm = os.FileMode(0664)

// See the tusd.DataStore interface for documentation about the different
// methods.
type FileStore struct {
	// Relative or absolute path to store files in. FileStore does not check
	// whether the path exists, use os.MkdirAll in this case on your own.
	Path string
}

// New creates a new file based storage backend. The directory specified will
// be used as the only storage entry. This method does not check
// whether the path exists, use os.MkdirAll to ensure.
// In addition, a locking mechanism is provided.
func New(path string) FileStore {
	return FileStore{path}
}

// UseIn sets this store as the core data store in the passed composer and adds
// all possible extension to it.
func (store FileStore) UseIn(composer *tusd.StoreComposer) {
	composer.UseCore(store)
	composer.UseGetReader(store)
	composer.UseTerminater(store)
	composer.UseLocker(store)
	composer.UseConcater(store)
}

func (store FileStore) NewUpload(info tusd.FileInfo) (id string, err error) {
	id = uid.Uid()
	info.ID = id

	// Create .bin file with no content
	file, err := os.OpenFile(store.binPath(id), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
	if err != nil {
		if os.IsNotExist(err) {
			err = fmt.Errorf("upload directory does not exist: %s", store.Path)
		}
		return "", err
	}
	defer file.Close()

	// writeInfo creates the file by itself if necessary
	err = store.writeInfo(id, info)
	return
}

func (store FileStore) WriteChunk(id string, offset int64, src io.Reader) (int64, error) {
	file, err := os.OpenFile(store.binPath(id), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return 0, err
	}
	defer file.Close()

	n, err := io.Copy(file, src)
	return n, err
}

func (store FileStore) GetInfo(id string) (tusd.FileInfo, error) {
	info := tusd.FileInfo{}
	data, err := ioutil.ReadFile(store.infoPath(id))
	if err != nil {
		return info, err
	}
	if err := json.Unmarshal(data, &info); err != nil {
		return info, err
	}

	stat, err := os.Stat(store.binPath(id))
	if err != nil {
		return info, err
	}

	info.Offset = stat.Size()

	return info, nil
}

func (store FileStore) GetReader(id string) (io.Reader, error) {
	return os.Open(store.binPath(id))
}

func (store FileStore) Terminate(id string) error {
	if err := os.Remove(store.infoPath(id)); err != nil {
		return err
	}
	if err := os.Remove(store.binPath(id)); err != nil {
		return err
	}
	return nil
}

func (store FileStore) ConcatUploads(dest string, uploads []string) (err error) {
	file, err := os.OpenFile(store.binPath(dest), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, id := range uploads {
		src, err := store.GetReader(id)
		if err != nil {
			return err
		}

		if _, err := io.Copy(file, src); err != nil {
			return err
		}
	}

	return
}

func (store FileStore) LockUpload(id string) error {
	lock, err := store.newLock(id)
	if err != nil {
		return err
	}

	err = lock.TryLock()
	if err == lockfile.ErrBusy {
		return tusd.ErrFileLocked
	}

	return err
}

func (store FileStore) UnlockUpload(id string) error {
	lock, err := store.newLock(id)
	if err != nil {
		return err
	}

	err = lock.Unlock()

	// A "no such file or directory" will be returned if no lockfile was found.
	// Since this means that the file has never been locked, we drop the error
	// and continue as if nothing happened.
	if os.IsNotExist(err) {
		err = nil
	}

	return err
}

// newLock contructs a new Lockfile instance.
func (store FileStore) newLock(id string) (lockfile.Lockfile, error) {
	path, err := filepath.Abs(filepath.Join(store.Path, id+".lock"))
	if err != nil {
		return lockfile.Lockfile(""), err
	}

	// We use Lockfile directly instead of lockfile.New to bypass the unnecessary
	// check whether the provided path is absolute since we just resolved it
	// on our own.
	return lockfile.Lockfile(path), nil
}

// binPath returns the path to the .bin storing the binary data.
func (store FileStore) binPath(id string) string {
	return filepath.Join(store.Path, id+".bin")
}

// infoPath returns the path to the .info file storing the file's info.
func (store FileStore) infoPath(id string) string {
	return filepath.Join(store.Path, id+".info")
}

// writeInfo updates the entire information. Everything will be overwritten.
func (store FileStore) writeInfo(id string, info tusd.FileInfo) error {
	data, err := json.Marshal(info)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(store.infoPath(id), data, defaultFilePerm)
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/github/hub/internal/assert"
	"testing"
)

func testPush(t *testing.T) {
	args := NewArgs([]string{"push", "origin,staging,qa", "bert_timeout"})
	push(nil, args)

	cmds := args.Commands()

	assert.Equal(t, 3, len(cmds))
	assert.Equal(t, "git push origin bert_timeout", cmds[0].String())
	assert.Equal(t, "git push staging bert_timeout", cmds[1].String())
}

func TestTransformPushArgs(t *testing.T) {
	args := NewArgs([]string{"push", "origin,staging,qa", "bert_timeout"})
	transformPushArgs(args)
	cmds := args.Commands()

	assert.Equal(t, 3, len(cmds))
	assert.Equal(t, "git push origin bert_timeout", cmds[0].String())
	assert.Equal(t, "git push staging bert_timeout", cmds[1].String())

	// TODO: travis-ci doesn't have HEAD
	//args = NewArgs([]string{"push", "origin"})
	//transformPushArgs(args)
	//cmds = args.Commands()

	//assert.Equal(t, 1, len(cmds))
	//pushRegexp := regexp.MustCompile("git push origin .+")
	//assert.T(t, pushRegexp.MatchString(cmds[0].String()))
}
<commit_msg>Remove unused func<commit_after>package commands

import (
	"github.com/github/hub/internal/assert"
	"testing"
)

func TestTransformPushArgs(t *testing.T) {
	args := NewArgs([]string{"push", "origin,staging,qa", "bert_timeout"})
	transformPushArgs(args)
	cmds := args.Commands()

	assert.Equal(t, 3, len(cmds))
	assert.Equal(t, "git push origin bert_timeout", cmds[0].String())
	assert.Equal(t, "git push staging bert_timeout", cmds[1].String())

	// TODO: travis-ci doesn't have HEAD
	//args = NewArgs([]string{"push", "origin"})
	//transformPushArgs(args)
	//cmds = args.Commands()

	//assert.Equal(t, 1, len(cmds))
	//pushRegexp := regexp.MustCompile("git push origin .+")
	//assert.T(t, pushRegexp.MatchString(cmds[0].String()))
}
<|endoftext|>


<commit_before>// Package alice implements a middleware chaining solution.
package alice

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

// A constructor for middleware
// that writes its own "tag" into the RW and does nothing else.
// Useful in checking if a chain is behaving in the right order.
func tagMiddleware(tag string) Constructor {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(tag))
			h.ServeHTTP(w, r)
		})
	}
}

// Not recommended (https://golang.org/pkg/reflect/#Value.Pointer),
// but the best we can do.
func funcsEqual(f1, f2 interface{}) bool {
	val1 := reflect.ValueOf(f1)
	val2 := reflect.ValueOf(f2)
	return val1.Pointer() == val2.Pointer()
}

var testApp = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("app\n"))
})

func TestThenWorksWithNoMiddleware(t *testing.T) {
	if !funcsEqual(New().Then(testApp), testApp) {
		t.Error("Then does not work with no middleware")
	}
}

func TestThenTreatsNilAsDefaultServeMux(t *testing.T) {
	if New().Then(nil) != http.DefaultServeMux {
		t.Error("Then does not treat nil as DefaultServeMux")
	}
}

func TestThenFuncTreatsNilAsDefaultServeMux(t *testing.T) {
	if New().ThenFunc(nil) != http.DefaultServeMux {
		t.Error("ThenFunc does not treat nil as DefaultServeMux")
	}
}

func TestThenFuncConstructsHandlerFunc(t *testing.T) {
	fn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
	})
	chained := New().ThenFunc(fn)
	rec := httptest.NewRecorder()

	chained.ServeHTTP(rec, (*http.Request)(nil))

	if reflect.TypeOf(chained) != reflect.TypeOf((http.HandlerFunc)(nil)) {
		t.Error("ThenFunc does not construct HandlerFunc")
	}
}

func TestThenOrdersHandlersCorrectly(t *testing.T) {
	t1 := tagMiddleware("t1\n")
	t2 := tagMiddleware("t2\n")
	t3 := tagMiddleware("t3\n")

	chained := New(t1, t2, t3).Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\napp\n" {
		t.Error("Then does not order handlers correctly")
	}
}

func TestAppendAddsHandlersCorrectly(t *testing.T) {
	chain := New(tagMiddleware("t1\n"), tagMiddleware("t2\n"))
	newChain := chain.Append(tagMiddleware("t3\n"), tagMiddleware("t4\n"))

	if len(chain.constructors) != 2 {
		t.Error("chain should have 2 constructors")
	}
	if len(newChain.constructors) != 4 {
		t.Error("newChain should have 4 constructors")
	}

	chained := newChain.Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\nt4\napp\n" {
		t.Error("Append does not add handlers correctly")
	}
}

func TestAppendRespectsImmutability(t *testing.T) {
	chain := New(tagMiddleware(""))
	newChain := chain.Append(tagMiddleware(""))

	if &chain.constructors[0] == &newChain.constructors[0] {
		t.Error("Apppend does not respect immutability")
	}
}

func TestExtendAddsHandlersCorrectly(t *testing.T) {
	chain1 := New(tagMiddleware("t1\n"), tagMiddleware("t2\n"))
	chain2 := New(tagMiddleware("t3\n"), tagMiddleware("t4\n"))
	newChain := chain1.Extend(chain2)

	if len(chain1.constructors) != 2 {
		t.Error("chain1 should contain 2 constructors")
	}
	if len(chain2.constructors) != 2 {
		t.Error("chain2 should contain 2 constructors")
	}
	if len(newChain.constructors) != 4 {
		t.Error("newChain should contain 4 constructors")
	}

	chained := newChain.Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\nt4\napp\n" {
		t.Error("Extend does not add handlers in correctly")
	}
}

func TestExtendRespectsImmutability(t *testing.T) {
	chain := New(tagMiddleware(""))
	newChain := chain.Extend(New(tagMiddleware("")))

	if &chain.constructors[0] == &newChain.constructors[0] {
		t.Error("Extend does not respect immutability")
	}
}
<commit_msg>Put back TestNew in chain_test.go<commit_after>// Package alice implements a middleware chaining solution.
package alice

import (
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
)

// A constructor for middleware
// that writes its own "tag" into the RW and does nothing else.
// Useful in checking if a chain is behaving in the right order.
func tagMiddleware(tag string) Constructor {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte(tag))
			h.ServeHTTP(w, r)
		})
	}
}

// Not recommended (https://golang.org/pkg/reflect/#Value.Pointer),
// but the best we can do.
func funcsEqual(f1, f2 interface{}) bool {
	val1 := reflect.ValueOf(f1)
	val2 := reflect.ValueOf(f2)
	return val1.Pointer() == val2.Pointer()
}

var testApp = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte("app\n"))
})

func TestNew(t *testing.T) {
	c1 := func(h http.Handler) http.Handler {
		return nil
	}

	c2 := func(h http.Handler) http.Handler {
		return http.StripPrefix("potato", nil)
	}

	slice := []Constructor{c1, c2}

	chain := New(slice...)
	for k := range slice {
		if !funcsEqual(chain.constructors[k], slice[k]) {
			t.Error("New does not add constructors correctly")
		}
	}
}

func TestThenWorksWithNoMiddleware(t *testing.T) {
	if !funcsEqual(New().Then(testApp), testApp) {
		t.Error("Then does not work with no middleware")
	}
}

func TestThenTreatsNilAsDefaultServeMux(t *testing.T) {
	if New().Then(nil) != http.DefaultServeMux {
		t.Error("Then does not treat nil as DefaultServeMux")
	}
}

func TestThenFuncTreatsNilAsDefaultServeMux(t *testing.T) {
	if New().ThenFunc(nil) != http.DefaultServeMux {
		t.Error("ThenFunc does not treat nil as DefaultServeMux")
	}
}

func TestThenFuncConstructsHandlerFunc(t *testing.T) {
	fn := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
	})
	chained := New().ThenFunc(fn)
	rec := httptest.NewRecorder()

	chained.ServeHTTP(rec, (*http.Request)(nil))

	if reflect.TypeOf(chained) != reflect.TypeOf((http.HandlerFunc)(nil)) {
		t.Error("ThenFunc does not construct HandlerFunc")
	}
}

func TestThenOrdersHandlersCorrectly(t *testing.T) {
	t1 := tagMiddleware("t1\n")
	t2 := tagMiddleware("t2\n")
	t3 := tagMiddleware("t3\n")

	chained := New(t1, t2, t3).Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\napp\n" {
		t.Error("Then does not order handlers correctly")
	}
}

func TestAppendAddsHandlersCorrectly(t *testing.T) {
	chain := New(tagMiddleware("t1\n"), tagMiddleware("t2\n"))
	newChain := chain.Append(tagMiddleware("t3\n"), tagMiddleware("t4\n"))

	if len(chain.constructors) != 2 {
		t.Error("chain should have 2 constructors")
	}
	if len(newChain.constructors) != 4 {
		t.Error("newChain should have 4 constructors")
	}

	chained := newChain.Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\nt4\napp\n" {
		t.Error("Append does not add handlers correctly")
	}
}

func TestAppendRespectsImmutability(t *testing.T) {
	chain := New(tagMiddleware(""))
	newChain := chain.Append(tagMiddleware(""))

	if &chain.constructors[0] == &newChain.constructors[0] {
		t.Error("Apppend does not respect immutability")
	}
}

func TestExtendAddsHandlersCorrectly(t *testing.T) {
	chain1 := New(tagMiddleware("t1\n"), tagMiddleware("t2\n"))
	chain2 := New(tagMiddleware("t3\n"), tagMiddleware("t4\n"))
	newChain := chain1.Extend(chain2)

	if len(chain1.constructors) != 2 {
		t.Error("chain1 should contain 2 constructors")
	}
	if len(chain2.constructors) != 2 {
		t.Error("chain2 should contain 2 constructors")
	}
	if len(newChain.constructors) != 4 {
		t.Error("newChain should contain 4 constructors")
	}

	chained := newChain.Then(testApp)

	w := httptest.NewRecorder()
	r, err := http.NewRequest("GET", "/", nil)
	if err != nil {
		t.Fatal(err)
	}

	chained.ServeHTTP(w, r)

	if w.Body.String() != "t1\nt2\nt3\nt4\napp\n" {
		t.Error("Extend does not add handlers in correctly")
	}
}

func TestExtendRespectsImmutability(t *testing.T) {
	chain := New(tagMiddleware(""))
	newChain := chain.Extend(New(tagMiddleware("")))

	if &chain.constructors[0] == &newChain.constructors[0] {
		t.Error("Extend does not respect immutability")
	}
}
<|endoftext|>


<commit_before>package gbench

import (
	"fmt"
	"math/rand"
	"strings"
	"time"

	rpc "github.com/relab/gorums/dev"
	"github.com/tylertreat/bench"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
)

// GrpcRequesterFactory implements RequesterFactory by creating a Requester which
// issues requests to a storage using the gRPC framework.
type GrpcRequesterFactory struct {
	Addrs             []string
	ReadQuorum        int
	WriteQuorum       int
	PayloadSize       int
	Timeout           time.Duration
	WriteRatioPercent int
	Concurrent        bool
}

// GetRequester returns a new Requester, called for each Benchmark connection.
func (r *GrpcRequesterFactory) GetRequester(uint64) bench.Requester {
	return &grpcRequester{
		addrs:       r.Addrs,
		readq:       r.ReadQuorum,
		writeq:      r.WriteQuorum,
		payloadSize: r.PayloadSize,
		timeout:     r.Timeout,
		writeRatio:  r.WriteRatioPercent,
		concurrent:  r.Concurrent,
		dialOpts: []grpc.DialOption{
			grpc.WithInsecure(),
			grpc.WithBlock(),
			grpc.WithTimeout(time.Second),
		},
	}
}

type client struct {
	conn   *grpc.ClientConn
	client rpc.RegisterClient
}

type grpcRequester struct {
	addrs       []string
	readq       int
	writeq      int
	payloadSize int
	timeout     time.Duration
	writeRatio  int
	concurrent  bool

	dialOpts []grpc.DialOption

	clients []*client

	state *rpc.State
	ctx   context.Context
}

func (gr *grpcRequester) Setup() error {
	gr.ctx = context.Background()
	gr.clients = make([]*client, len(gr.addrs))

	for i := 0; i < len(gr.clients); i++ {
		conn, err := grpc.Dial(gr.addrs[i], gr.dialOpts...)
		if err != nil {
			return fmt.Errorf("error connecting to %q: %v", gr.addrs[i], err)
		}
		gr.clients[i] = &client{
			conn:   conn,
			client: rpc.NewRegisterClient(conn),
		}
	}

	// Set initial state.
	gr.state = &rpc.State{
		Value:     strings.Repeat("x", gr.payloadSize),
		Timestamp: time.Now().UnixNano(),
	}

	for i, c := range gr.clients {
		wreply, err := c.client.Write(gr.ctx, gr.state)
		if err != nil {
			return fmt.Errorf("%s: write rpc error: %v", gr.addrs[i], err)
		}
		if !wreply.New {
			return fmt.Errorf("%s: intital write reply was not marked as new", gr.addrs[i])
		}
	}

	return nil
}

func (gr *grpcRequester) Request() error {
	write := gr.doWrite()
	if gr.concurrent {
		return gr.concurrentReq(write)
	}
	return gr.singleReq(write)
}

func (gr *grpcRequester) singleReq(write bool) error {
	client := gr.clients[0].client
	if !write {
		_, err := client.Read(gr.ctx, &rpc.ReadRequest{})
		return err
	}
	gr.state.Timestamp = time.Now().UnixNano()
	_, err := client.Write(gr.ctx, gr.state)
	return err
}

func (gr *grpcRequester) concurrentReq(write bool) error {
	if write {
		_, err := gr.writeConcurrent()
		return err
	}
	_, err := gr.readConcurrent()
	return err
}

func (gr *grpcRequester) writeConcurrent() (*rpc.WriteResponse, error) {
	replies := make(chan *rpc.WriteResponse, len(gr.clients))
	for _, c := range gr.clients {
		go func(c *client) {
			gr.state.Timestamp = time.Now().UnixNano()
			rep, err := c.client.Write(gr.ctx, gr.state)
			if err != nil {
				panic("write error")
			}
			replies <- rep
		}(c)
	}

	count := 0
	for reply := range replies {
		count++
		if count >= gr.writeq {
			return reply, nil
		}
	}

	return nil, fmt.Errorf("write incomplete")
}

func (gr *grpcRequester) readConcurrent() (*rpc.State, error) {
	replies := make(chan *rpc.State, len(gr.clients))
	for _, c := range gr.clients {
		go func(c *client) {
			rep, err := c.client.Read(gr.ctx, &rpc.ReadRequest{})
			if err != nil {
				panic("read error")
			}
			replies <- rep
		}(c)
	}

	count := 0
	for reply := range replies {
		count++
		if count >= gr.readq {
			return reply, nil
		}
	}

	return nil, fmt.Errorf("read incomplete")
}

func (gr *grpcRequester) Teardown() error {
	for _, c := range gr.clients {
		_ = c.conn.Close()
		c.conn = nil
	}
	return nil
}

func (gr *grpcRequester) doWrite() bool {
	switch gr.writeRatio {
	case 0:
		return false
	case 100:
		return true
	default:
		x := rand.Intn(100)
		if x < gr.writeRatio {
			return true
		}
		return false
	}
}
<commit_msg>gbench: update after Register->Storage dev renaming<commit_after>package gbench

import (
	"fmt"
	"math/rand"
	"strings"
	"time"

	rpc "github.com/relab/gorums/dev"
	"github.com/tylertreat/bench"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
)

// GrpcRequesterFactory implements RequesterFactory by creating a Requester which
// issues requests to a storage using the gRPC framework.
type GrpcRequesterFactory struct {
	Addrs             []string
	ReadQuorum        int
	WriteQuorum       int
	PayloadSize       int
	Timeout           time.Duration
	WriteRatioPercent int
	Concurrent        bool
}

// GetRequester returns a new Requester, called for each Benchmark connection.
func (r *GrpcRequesterFactory) GetRequester(uint64) bench.Requester {
	return &grpcRequester{
		addrs:       r.Addrs,
		readq:       r.ReadQuorum,
		writeq:      r.WriteQuorum,
		payloadSize: r.PayloadSize,
		timeout:     r.Timeout,
		writeRatio:  r.WriteRatioPercent,
		concurrent:  r.Concurrent,
		dialOpts: []grpc.DialOption{
			grpc.WithInsecure(),
			grpc.WithBlock(),
			grpc.WithTimeout(time.Second),
		},
	}
}

type client struct {
	conn   *grpc.ClientConn
	client rpc.StorageClient
}

type grpcRequester struct {
	addrs       []string
	readq       int
	writeq      int
	payloadSize int
	timeout     time.Duration
	writeRatio  int
	concurrent  bool

	dialOpts []grpc.DialOption

	clients []*client

	state *rpc.State
	ctx   context.Context
}

func (gr *grpcRequester) Setup() error {
	gr.ctx = context.Background()
	gr.clients = make([]*client, len(gr.addrs))

	for i := 0; i < len(gr.clients); i++ {
		conn, err := grpc.Dial(gr.addrs[i], gr.dialOpts...)
		if err != nil {
			return fmt.Errorf("error connecting to %q: %v", gr.addrs[i], err)
		}
		gr.clients[i] = &client{
			conn:   conn,
			client: rpc.NewStorageClient(conn),
		}
	}

	// Set initial state.
	gr.state = &rpc.State{
		Value:     strings.Repeat("x", gr.payloadSize),
		Timestamp: time.Now().UnixNano(),
	}

	for i, c := range gr.clients {
		wreply, err := c.client.Write(gr.ctx, gr.state)
		if err != nil {
			return fmt.Errorf("%s: write rpc error: %v", gr.addrs[i], err)
		}
		if !wreply.New {
			return fmt.Errorf("%s: intital write reply was not marked as new", gr.addrs[i])
		}
	}

	return nil
}

func (gr *grpcRequester) Request() error {
	write := gr.doWrite()
	if gr.concurrent {
		return gr.concurrentReq(write)
	}
	return gr.singleReq(write)
}

func (gr *grpcRequester) singleReq(write bool) error {
	client := gr.clients[0].client
	if !write {
		_, err := client.Read(gr.ctx, &rpc.ReadRequest{})
		return err
	}
	gr.state.Timestamp = time.Now().UnixNano()
	_, err := client.Write(gr.ctx, gr.state)
	return err
}

func (gr *grpcRequester) concurrentReq(write bool) error {
	if write {
		_, err := gr.writeConcurrent()
		return err
	}
	_, err := gr.readConcurrent()
	return err
}

func (gr *grpcRequester) writeConcurrent() (*rpc.WriteResponse, error) {
	replies := make(chan *rpc.WriteResponse, len(gr.clients))
	for _, c := range gr.clients {
		go func(c *client) {
			gr.state.Timestamp = time.Now().UnixNano()
			rep, err := c.client.Write(gr.ctx, gr.state)
			if err != nil {
				panic("write error")
			}
			replies <- rep
		}(c)
	}

	count := 0
	for reply := range replies {
		count++
		if count >= gr.writeq {
			return reply, nil
		}
	}

	return nil, fmt.Errorf("write incomplete")
}

func (gr *grpcRequester) readConcurrent() (*rpc.State, error) {
	replies := make(chan *rpc.State, len(gr.clients))
	for _, c := range gr.clients {
		go func(c *client) {
			rep, err := c.client.Read(gr.ctx, &rpc.ReadRequest{})
			if err != nil {
				panic("read error")
			}
			replies <- rep
		}(c)
	}

	count := 0
	for reply := range replies {
		count++
		if count >= gr.readq {
			return reply, nil
		}
	}

	return nil, fmt.Errorf("read incomplete")
}

func (gr *grpcRequester) Teardown() error {
	for _, c := range gr.clients {
		_ = c.conn.Close()
		c.conn = nil
	}
	return nil
}

func (gr *grpcRequester) doWrite() bool {
	switch gr.writeRatio {
	case 0:
		return false
	case 100:
		return true
	default:
		x := rand.Intn(100)
		if x < gr.writeRatio {
			return true
		}
		return false
	}
}
<|endoftext|>