<commit_before>package main

import (
  "path"
  "os"
  "sync"
  "log"
  "sort"
  "io"
  "io/ioutil"
  "errors"
  "path/filepath"
)

const (
  CopyPrice = 100
  RenamePrice = 10

  RemoveBackupFactor = 50
  RemoveFactor = RenamePrice
  UpdateFactor = RenamePrice + CopyPrice
  AddFactor = CopyPrice
)

type BackupPair struct {
  relpath string
  newpath string
}

type ProgressHandler interface {
  HandleSystemMessage(message string)
  HandlePercentChange(percent int)
  HandleFinish()
}

type LogProgressHandler struct {
}

type ProgressReporter struct {
  grandTotal uint64
  currentProgress uint64
  progressChan chan int64
  percent int //0..100
  reportingChan chan bool
  systemMessageChan chan string
  finished chan bool
  progressHandler ProgressHandler
}

type PackageInstaller struct {
  backups map[string]string
  backupsChan chan BackupPair
  progressReporter *ProgressReporter
  installDir string
  packageDir string
  failInTheEnd bool // for debugging purposes
}

func (pi *PackageInstaller) Install(filesProvider UpdateFilesProvider) error {
  pi.progressReporter.grandTotal = pi.calculateGrandTotals(filesProvider)
  go pi.progressReporter.reportingLoop()
  defer close(pi.progressReporter.progressChan)
  defer func() {
    go func () {
      pi.progressReporter.finished <- true
    }()
  }()

  err := pi.installPackage(filesProvider)

  if err == nil {
    pi.afterSuccess()
  } else {
    pi.afterFailure(filesProvider)
  }

  return err
}

func (pi *PackageInstaller) calculateGrandTotals(filesProvider UpdateFilesProvider) uint64 {
  var sum uint64

  for _, fi := range filesProvider.FilesToRemove() {
    sum += uint64(fi.FileSize * RemoveFactor) / 100
    sum += uint64(RemoveBackupFactor)
  }

  for _, fi := range filesProvider.FilesToUpdate() {
    sum += uint64(fi.FileSize * UpdateFactor) / 100
    sum += uint64(RemoveBackupFactor)
  }

  for _, fi := range filesProvider.FilesToAdd() {
    sum += uint64(fi.FileSize * AddFactor) / 100
  }

  return sum
}

func (pi *PackageInstaller) installPackage(filesProvider UpdateFilesProvider) (err error) {
  log.Println("Installing package...")

  var wg sync.WaitGroup
  wg.Add(1)
  go func() {
    for bp := range pi.backupsChan {
      pi.backups[bp.relpath] = bp.newpath
    }
    wg.Done()
  }()

  pi.progressReporter.systemMessageChan <- "Removing components"
  err = pi.removeFiles(filesProvider.FilesToRemove())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Updating components"
  err = pi.updateFiles(filesProvider.FilesToUpdate())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Adding components"
  err = pi.addFiles(filesProvider.FilesToAdd())
  if err != nil {
    return err
  }

  go func() {
    close(pi.backupsChan)
  }()

  wg.Wait()

  if pi.failInTheEnd {
    err = errors.New("Fail by demand")
  }

  return err
}

func (pi *PackageInstaller) afterSuccess() {
  log.Println("After success")
  pi.progressReporter.systemMessageChan <- "Finishing the installation..."
  pi.removeBackups();
  cleanupEmptyDirs(pi.installDir)
}

func (pi *PackageInstaller) afterFailure(filesProvider UpdateFilesProvider) {
  log.Println("After failure")
  pi.progressReporter.systemMessageChan <- "Cleaning up..."
  purgeFiles(pi.installDir, filesProvider.FilesToAdd())
  pi.restoreBackups()
  pi.removeBackups()
  cleanupEmptyDirs(pi.installDir)
}

func copyFile(src, dst string) (err error) {
  in, err := os.Open(src)
  if err != nil {
    log.Printf("Failed to open source: %v", err)
    return
  }

  defer in.Close()

  out, err := os.Create(dst)
  if err != nil {
    log.Printf("Failed to create destination: %v", err)
    return
  }

  defer func() {
    cerr := out.Close()
    if err == nil {
      err = cerr
    }
  }()

  if _, err = io.Copy(out, in); err != nil {
    return
  }

  err = out.Sync()
  return
}

func (pi *PackageInstaller) backupFile(relpath string) error {
  log.Printf("Backing up %v", relpath)

  oldpath := path.Join(pi.installDir, relpath)
  backupPath := relpath + ".bak"

  newpath := path.Join(pi.installDir, backupPath)

  err := os.Rename(oldpath, newpath)

  if err == nil {
    pi.backupsChan <- BackupPair{relpath: relpath, newpath: newpath}
  } else {
    log.Printf("Backup failed: %v", err)
  }

  return err
}

func (pi *PackageInstaller) restoreBackups() {
  log.Printf("Restoring %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for relpath, backuppath := range pi.backups {
    wg.Add(1)

    relativePath := relpath
    pathToRestore := backuppath

    go func() {
      defer wg.Done()

      oldpath := path.Join(pi.installDir, relativePath)
      err := os.Rename(pathToRestore, oldpath)

      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeBackups() {
  log.Printf("Removing %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for _, backuppath := range pi.backups {
    wg.Add(1)

    pathToRemove := backuppath

    go func() {
      defer wg.Done()

      err := os.Remove(pathToRemove)
      if err != nil {
        log.Println(err)
      }

      go pi.progressReporter.accountBackupRemove()
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeFiles(files []*UpdateFileInfo) error {
  log.Printf("Removing %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)
    pathToRemove, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      fullpath := filepath.Join(pi.installDir, pathToRemove)
      log.Printf("Removing file %v", fullpath)

      err := pi.backupFile(pathToRemove)

      if err != nil {
        log.Printf("Removing file %v failed", pathToRemove)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountRemove(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) updateFiles(files []*UpdateFileInfo) error {
  log.Printf("Updating %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToUpdate, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToUpdate)
      log.Printf("Updating file %v", oldpath)

      err := pi.backupFile(pathToUpdate)

      if err == nil {
        newpath := path.Join(pi.packageDir, pathToUpdate)
        err = os.Rename(newpath, oldpath)
      }

      if err != nil {
        log.Printf("Updating file %v failed", pathToUpdate)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountUpdate(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) addFiles(files []*UpdateFileInfo) error {
  log.Printf("Adding %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToAdd, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToAdd)
      ensureDirExists(oldpath)

      newpath := path.Join(pi.packageDir, pathToAdd)
      err := os.Rename(newpath, oldpath)

      if err != nil {
        log.Printf("Adding file %v failed", pathToAdd)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountAdd(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func purgeFiles(root string, files []*UpdateFileInfo) {
  log.Printf("Purging %v files", len(files))

  var wg sync.WaitGroup

  for _, fi := range files {
    wg.Add(1)

    fileToPurge := fi.Filepath

    go func() {
      defer wg.Done()

      fullpath := path.Join(root, fileToPurge)
      err := os.Remove(fullpath)
      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func ensureDirExists(fullpath string) (err error) {
  dirpath := path.Dir(fullpath)
  err = os.MkdirAll(dirpath, os.ModeDir)
  if err != nil {
    log.Printf("Failed to create directory %v", dirpath)
  }

  return err
}

type ByLength []string

func (s ByLength) Len() int {
    return len(s)
}
func (s ByLength) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByLength) Less(i, j int) bool {
    return len(s[i]) > len(s[j])
}

func cleanupEmptyDirs(root string) {
  c := make(chan string)

  go func() {
    var wg sync.WaitGroup
    err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
      if err != nil {
        return err
      }

      if info.Mode().IsDir() {
        wg.Add(1)
        go func() {
          c <- path
          wg.Done()
        }()
      }

      return nil
    })

    if err != nil {
      log.Println(err)
    }

    go func() {
      wg.Wait()
      close(c)
    }()
  }()

  dirs := make([]string, 0)
  for path := range c {
    dirs = append(dirs, path)
  }

  removeEmptyDirs(dirs)
}

func removeEmptyDirs(dirs []string) {
  sort.Sort(ByLength(dirs))

  for _, dirpath := range dirs {
    entries, err := ioutil.ReadDir(dirpath)
    if err != nil { continue }

    if len(entries) == 0 {
      log.Printf("Removing empty dir %v", dirpath)

      err = os.Remove(dirpath)
      if err != nil {
        log.Println(err)
      }
    }
  }
}

func (pr *ProgressReporter) accountRemove(progress int64) {
  pr.progressChan <- (progress*RemoveFactor)/100
}

func (pr *ProgressReporter) accountUpdate(progress int64) {
  pr.progressChan <- (progress*UpdateFactor)/100
}

func (pr *ProgressReporter) accountAdd(progress int64) {
  pr.progressChan <- (progress*AddFactor)/100
}

func (pr *ProgressReporter) accountBackupRemove() {
  pr.progressChan <- RemoveBackupFactor
}

func (pr *ProgressReporter) reportingLoop() {
  for chunk := range pr.progressChan {
    pr.currentProgress += uint64(chunk)

    percent := (pr.currentProgress*100) / pr.grandTotal
    pr.percent = int(percent)

    go func() {
      pr.reportingChan <- true
    }()
  }

  close(pr.reportingChan)
}

func (pr *ProgressReporter) receiveUpdates() {
  for _ = range pr.reportingChan {
    pr.progressHandler.HandlePercentChange(pr.percent)
  }
}

func (pr *ProgressReporter) receiveSystemMessages() {
  for msg := range pr.systemMessageChan {
    pr.progressHandler.HandleSystemMessage(msg)
  }
}

func (pr *ProgressReporter) receiveFinish() {
  <- pr.finished
  pr.progressHandler.HandleFinish()
}

func (pr *ProgressReporter) handleProgress() {
  go pr.receiveSystemMessages()
  go pr.receiveUpdates()
  go pr.receiveFinish()
}

func (ph *LogProgressHandler) HandlePercentChange(percent int) {
  log.Printf("Completed %v%%", percent)
}

func (ph *LogProgressHandler) HandleSystemMessage(msg string) {
  log.Printf("System message: %v", msg)
}

func (ph *LogProgressHandler) HandleFinish() {
  log.Printf("Finished")
}
<commit_msg>Minor fixes<commit_after>package main

import (
  "path"
  "os"
  "sync"
  "log"
  "sort"
  "io"
  "io/ioutil"
  "errors"
  "path/filepath"
)

const (
  CopyPrice = 100
  RenamePrice = 10

  RemoveBackupPrice = 1000
  RemoveFactor = RenamePrice
  UpdateFactor = RenamePrice + CopyPrice
  AddFactor = CopyPrice
)

type BackupPair struct {
  relpath string
  newpath string
}

type ProgressHandler interface {
  HandleSystemMessage(message string)
  HandlePercentChange(percent int)
  HandleFinish()
}

type LogProgressHandler struct {
}

type ProgressReporter struct {
  grandTotal uint64
  currentProgress uint64
  progressChan chan int64
  percent int //0..100
  reportingChan chan bool
  systemMessageChan chan string
  finished chan bool
  progressHandler ProgressHandler
}

type PackageInstaller struct {
  backups map[string]string
  backupsChan chan BackupPair
  progressReporter *ProgressReporter
  installDir string
  packageDir string
  failInTheEnd bool // for debugging purposes
}

func (pi *PackageInstaller) Install(filesProvider UpdateFilesProvider) error {
  pi.progressReporter.grandTotal = pi.calculateGrandTotals(filesProvider)
  go pi.progressReporter.reportingLoop()
  defer close(pi.progressReporter.progressChan)
  defer func() {
    go func () {
      pi.progressReporter.finished <- true
    }()
  }()

  err := pi.installPackage(filesProvider)

  if err == nil {
    pi.afterSuccess()
  } else {
    pi.afterFailure(filesProvider)
  }

  return err
}

func (pi *PackageInstaller) calculateGrandTotals(filesProvider UpdateFilesProvider) uint64 {
  var sum uint64

  for _, fi := range filesProvider.FilesToRemove() {
    sum += uint64(fi.FileSize * RemoveFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToUpdate() {
    sum += uint64(fi.FileSize * UpdateFactor) / 100
    sum += uint64(RemoveBackupPrice)
  }

  for _, fi := range filesProvider.FilesToAdd() {
    sum += uint64(fi.FileSize * AddFactor) / 100
  }

  return sum
}

func (pi *PackageInstaller) installPackage(filesProvider UpdateFilesProvider) (err error) {
  log.Println("Installing package...")

  var wg sync.WaitGroup
  wg.Add(1)
  go func() {
    for bp := range pi.backupsChan {
      pi.backups[bp.relpath] = bp.newpath
    }
    wg.Done()
  }()

  pi.progressReporter.systemMessageChan <- "Removing components"
  err = pi.removeFiles(filesProvider.FilesToRemove())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Updating components"
  err = pi.updateFiles(filesProvider.FilesToUpdate())
  if err != nil {
    return err
  }

  pi.progressReporter.systemMessageChan <- "Adding components"
  err = pi.addFiles(filesProvider.FilesToAdd())
  if err != nil {
    return err
  }

  go func() {
    close(pi.backupsChan)
  }()

  wg.Wait()

  if pi.failInTheEnd {
    err = errors.New("Fail by demand")
  }

  return err
}

func (pi *PackageInstaller) afterSuccess() {
  log.Println("After success")
  pi.progressReporter.systemMessageChan <- "Finishing the installation..."
  pi.removeBackups();
  cleanupEmptyDirs(pi.installDir)
}

func (pi *PackageInstaller) afterFailure(filesProvider UpdateFilesProvider) {
  log.Println("After failure")
  pi.progressReporter.systemMessageChan <- "Cleaning up..."
  purgeFiles(pi.installDir, filesProvider.FilesToAdd())
  pi.restoreBackups()
  pi.removeBackups()
  cleanupEmptyDirs(pi.installDir)
}

func copyFile(src, dst string) (err error) {
  in, err := os.Open(src)
  if err != nil {
    log.Printf("Failed to open source: %v", err)
    return
  }

  defer in.Close()

  out, err := os.Create(dst)
  if err != nil {
    log.Printf("Failed to create destination: %v", err)
    return
  }

  defer func() {
    cerr := out.Close()
    if err == nil {
      err = cerr
    }
  }()

  if _, err = io.Copy(out, in); err != nil {
    return
  }

  err = out.Sync()
  return
}

func (pi *PackageInstaller) backupFile(relpath string) error {
  log.Printf("Backing up %v", relpath)

  oldpath := path.Join(pi.installDir, relpath)
  backupPath := relpath + ".bak"

  newpath := path.Join(pi.installDir, backupPath)

  err := os.Rename(oldpath, newpath)

  if err == nil {
    pi.backupsChan <- BackupPair{relpath: relpath, newpath: newpath}
  } else {
    log.Printf("Backup failed: %v", err)
  }

  return err
}

func (pi *PackageInstaller) restoreBackups() {
  log.Printf("Restoring %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for relpath, backuppath := range pi.backups {
    wg.Add(1)

    relativePath := relpath
    pathToRestore := backuppath

    go func() {
      defer wg.Done()

      oldpath := path.Join(pi.installDir, relativePath)
      err := os.Rename(pathToRestore, oldpath)

      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeBackups() {
  log.Printf("Removing %v backups", len(pi.backups))

  var wg sync.WaitGroup

  for _, backuppath := range pi.backups {
    wg.Add(1)

    pathToRemove := backuppath

    go func() {
      defer wg.Done()

      err := os.Remove(pathToRemove)
      if err != nil {
        log.Println(err)
      }

      go pi.progressReporter.accountBackupRemove()
    }()
  }

  wg.Wait()
}

func (pi *PackageInstaller) removeFiles(files []*UpdateFileInfo) error {
  log.Printf("Removing %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)
    pathToRemove, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      fullpath := filepath.Join(pi.installDir, pathToRemove)
      log.Printf("Removing file %v", fullpath)

      err := pi.backupFile(pathToRemove)

      if err != nil {
        log.Printf("Removing file %v failed", pathToRemove)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountRemove(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) updateFiles(files []*UpdateFileInfo) error {
  log.Printf("Updating %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToUpdate, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToUpdate)
      log.Printf("Updating file %v", oldpath)

      err := pi.backupFile(pathToUpdate)

      if err == nil {
        newpath := path.Join(pi.packageDir, pathToUpdate)
        err = os.Rename(newpath, oldpath)
      }

      if err != nil {
        log.Printf("Updating file %v failed", pathToUpdate)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountUpdate(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func (pi *PackageInstaller) addFiles(files []*UpdateFileInfo) error {
  log.Printf("Adding %v files", len(files))

  var wg sync.WaitGroup
  errc := make(chan error)
  done := make(chan bool)

  for _, fi := range files {
    wg.Add(1)

    pathToAdd, filesize := fi.Filepath, fi.FileSize

    go func() {
      defer wg.Done()

      select {
      case <-done: return
      default:
      }

      oldpath := path.Join(pi.installDir, pathToAdd)
      ensureDirExists(oldpath)

      newpath := path.Join(pi.packageDir, pathToAdd)
      err := os.Rename(newpath, oldpath)

      if err != nil {
        log.Printf("Adding file %v failed", pathToAdd)
        log.Println(err)
        errc <- err
        close(done)
      } else {
        go pi.progressReporter.accountAdd(filesize)
      }
    }()
  }

  go func() {
    errc <- nil
  }()

  wg.Wait()

  if err := <-errc; err != nil {
    return err
  }

  return nil
}

func purgeFiles(root string, files []*UpdateFileInfo) {
  log.Printf("Purging %v files", len(files))

  var wg sync.WaitGroup

  for _, fi := range files {
    wg.Add(1)

    fileToPurge := fi.Filepath

    go func() {
      defer wg.Done()

      fullpath := path.Join(root, fileToPurge)
      err := os.Remove(fullpath)
      if err != nil {
        log.Println(err)
      }
    }()
  }

  wg.Wait()
}

func ensureDirExists(fullpath string) (err error) {
  dirpath := path.Dir(fullpath)
  err = os.MkdirAll(dirpath, os.ModeDir)
  if err != nil {
    log.Printf("Failed to create directory %v", dirpath)
  }

  return err
}

type ByLength []string

func (s ByLength) Len() int {
    return len(s)
}
func (s ByLength) Swap(i, j int) {
    s[i], s[j] = s[j], s[i]
}
func (s ByLength) Less(i, j int) bool {
    return len(s[i]) > len(s[j])
}

func cleanupEmptyDirs(root string) {
  c := make(chan string)

  go func() {
    var wg sync.WaitGroup
    err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
      if err != nil {
        return err
      }

      if info.Mode().IsDir() {
        wg.Add(1)
        go func() {
          c <- path
          wg.Done()
        }()
      }

      return nil
    })

    if err != nil {
      log.Println(err)
    }

    go func() {
      wg.Wait()
      close(c)
    }()
  }()

  dirs := make([]string, 0)
  for path := range c {
    dirs = append(dirs, path)
  }

  removeEmptyDirs(dirs)
}

func removeEmptyDirs(dirs []string) {
  sort.Sort(ByLength(dirs))

  for _, dirpath := range dirs {
    entries, err := ioutil.ReadDir(dirpath)
    if err != nil { continue }

    if len(entries) == 0 {
      log.Printf("Removing empty dir %v", dirpath)

      err = os.Remove(dirpath)
      if err != nil {
        log.Println(err)
      }
    }
  }
}

func (pr *ProgressReporter) accountRemove(progress int64) {
  pr.progressChan <- (progress*RemoveFactor)/100
}

func (pr *ProgressReporter) accountUpdate(progress int64) {
  pr.progressChan <- (progress*UpdateFactor)/100
}

func (pr *ProgressReporter) accountAdd(progress int64) {
  pr.progressChan <- (progress*AddFactor)/100
}

func (pr *ProgressReporter) accountBackupRemove() {
  // ability to know precise size incomplicates this
  pr.progressChan <- RemoveBackupPrice
}

func (pr *ProgressReporter) reportingLoop() {
  for chunk := range pr.progressChan {
    pr.currentProgress += uint64(chunk)

    percent := (pr.currentProgress*100) / pr.grandTotal
    pr.percent = int(percent)

    go func() {
      pr.reportingChan <- true
    }()
  }

  close(pr.reportingChan)
}

func (pr *ProgressReporter) receiveUpdates() {
  for _ = range pr.reportingChan {
    pr.progressHandler.HandlePercentChange(pr.percent)
  }
}

func (pr *ProgressReporter) receiveSystemMessages() {
  for msg := range pr.systemMessageChan {
    pr.progressHandler.HandleSystemMessage(msg)
  }
}

func (pr *ProgressReporter) receiveFinish() {
  <- pr.finished
  pr.progressHandler.HandleFinish()
}

func (pr *ProgressReporter) handleProgress() {
  go pr.receiveSystemMessages()
  go pr.receiveUpdates()
  go pr.receiveFinish()
}

func (ph *LogProgressHandler) HandlePercentChange(percent int) {
  log.Printf("Completed %v%%", percent)
}

func (ph *LogProgressHandler) HandleSystemMessage(msg string) {
  log.Printf("System message: %v", msg)
}

func (ph *LogProgressHandler) HandleFinish() {
  log.Printf("Finished")
}
<|endoftext|>


<commit_before>/*
	Copyright 2017 by Christian Hüning (christianhuening@googlemail.com).

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at
		http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
*/

package usecases

import (
	"fmt"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/gitlabclient"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/k8sclient"
	"log"
	"os"
	"strconv"
	"sync"
	"time"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/graylog"
)

/*
What to fetch from k8s api
- Get all namespaces with gitlab-origin field (ns without that field won't be gitlab created)
- Get all rolebindings of these namespaces

What to get from gitlab
- get all groups
- get all projects
- get all users (private namespace)

Algo:
1. Delete all namespaces which are not in the gitlab Set
2. Iterate all gitlab namespaces
   if namespace is present in k8s set:
	2.1 Iterate all rolebindings
	2.2 Compare to rolebindings from k8s set by using the gitlab-origin field as key and
		2.2.1 Delete every rolebinding not present in the gitlab set
		2.2.1 Create every rolebinding not present in the k8s set
   else:
	2.1 Create namespace
		2.1.1 If namespace is present by name, but does not have a gitlab-origin label attached
		AND is not(!) labeled with 'gitlab-ignored' it get's labeled with its origin name.
		Otherwise the naming collision is solved by suffixing the name with a counter
	2.2 Create all rolebindings

 done

*/

// TODO : Cache Webhooks while Sync is running and execute them later!

func PerformGlK8sSync() {
	log.Println("Starting new Synchronization run!")
	log.Println("Getting Gitlab Contents...")
	gitlabContent, err := gitlabclient.GetFullGitlabContent()
	if check(err) {
		return
	}

	// 1. delete all Namespaces which are not in the gitlab set
	log.Println("Getting K8s Contents...")
	gitlabNamespacesInK8s := k8sclient.GetAllGitlabOriginNamesFromNamespacesWithOriginLabel()

	log.Println("Deleting all namespaces which are no longer in the gitlab namespace...")
	for _, originalName := range gitlabNamespacesInK8s {
		delete := true

		for _, user := range gitlabContent.Users {
			if originalName == user.Username {
				delete = false
				break
			}
		}

		if delete {
			for _, project := range gitlabContent.Projects {
				if originalName == project.PathWithNameSpace {
					delete = false
					break
				}
			}
		}

		if delete {
			for _, group := range gitlabContent.Groups {
				if originalName == group.FullPath {
					delete = false
					break
				}
			}
		}

		if delete {
			actualNs := k8sclient.DeleteNamespace(originalName)
			graylog.DeleteStream(actualNs)
		}
	}

	log.Println("Reading custom-rolebindings if any...")

	cRaB := ReadAndApplyCustomRolesAndBindings()

	var syncDoneWg sync.WaitGroup
	syncDoneWg.Add(3)

	log.Println("Syncing Gitlab Users...")
	go syncUsers(gitlabContent, cRaB, &syncDoneWg)

	log.Println("Syncing Gitlab Groups...")
	go syncGroups(gitlabContent, cRaB, &syncDoneWg)

	log.Println("Syncing Gitlab Projects...")
	go syncProjects(gitlabContent, cRaB, &syncDoneWg)

	syncDoneWg.Wait()
	log.Println("Finished Synchronization run.")
}

func syncUsers(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	for _, user := range gitlabContent.Users {
		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(user.Username)
		if actualNamespace != "" {

			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)
			roleName := k8sclient.GetGroupRoleName("Master")
			expectedGitlabRolebindingName := k8sclient.ConstructRoleBindingName(user.Username, roleName, actualNamespace)

			// 2.1 Iterate all roleBindings
			for rb := range k8sRoleBindings {
				if rb != expectedGitlabRolebindingName && !cRaB.RoleBindings[rb] {
					k8sclient.DeleteGroupRoleBindingByName(rb, actualNamespace)
				}
			}
			// make sure the project's role binding is present
			if !k8sRoleBindings[expectedGitlabRolebindingName] {
				k8sclient.CreateGroupRoleBinding(user.Username, user.Username, "Master")
			}

			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)

		} else {
			// create Namespace & RoleBinding
			k8sclient.CreateNamespace(user.Username)
			k8sclient.CreateGroupRoleBinding(user.Username, user.Username, "Master")
		}
	}
}

func syncGroups(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	// same same for Groups
	for _, group := range gitlabContent.Groups {
		if group.FullPath == "kube-system" {
			continue
		} // ignore kube-system group

		if debugSync() {
			log.Println("Syncing: " + group.FullPath)
		}

		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(group.FullPath)
		if debugSync() {
			log.Println("ActualNamespace: " + actualNamespace)
		}
		if actualNamespace != "" {
			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)
			if debugSync() {
				log.Printf("Found %d rolebindings \n", len(k8sRoleBindings))
			}

			// get expectedRoleBindings by retrieved Members
			expectedRoleBindings := map[string]bool{}

			// create or get ServiceAccount
			_, roleBindingName, err := k8sclient.CreateServiceAccountAndRoleBinding(group.FullPath)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount for group %s. Err was: %s", group.FullPath, err))
			}
			expectedRoleBindings[roleBindingName] = true

			for _, member := range group.Members {
				if debugSync() {
					log.Println("Processing member " + member.Name)
				}
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				roleName := k8sclient.GetGroupRoleName(accessLevel)
				rbName := k8sclient.ConstructRoleBindingName(member.Username, roleName, actualNamespace)
				expectedRoleBindings[rbName] = true

				if debugSync() {
					log.Printf("AccessLevel: %s, roleName: %s, rbName: %s", accessLevel, roleName, rbName)
				}

				// make sure the groups's expected rolebindings are present
				if !k8sRoleBindings[rbName] {
					if debugSync() {
						log.Println("Creating RoleBinding " + rbName)
					}
					k8sclient.CreateGroupRoleBinding(member.Username, group.FullPath, accessLevel)
				}
			}

			// 2.1 Iterate all roleBindings and delete those which are not anymore present in gitlab or in custom roles
			for rb := range k8sRoleBindings {
				if !expectedRoleBindings[rb] && !cRaB.RoleBindings[rb] {
					if debugSync() {
						log.Println("Deleting RoleBinding " + rb)
					}
					k8sclient.DeleteGroupRoleBindingByName(rb, actualNamespace)
				}
			}
			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)

		} else {
			// create Namespace & RoleBinding
			k8sclient.CreateNamespace(group.FullPath)
			_, _, err := k8sclient.CreateServiceAccountAndRoleBinding(group.FullPath)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount for group %s. Err was: %s", group.FullPath, err))
			}
			if debugSync() {
				log.Println("Creating Namespace for " + group.FullPath)
			}
			for _, member := range group.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				k8sclient.CreateGroupRoleBinding(member.Username, group.FullPath, accessLevel)
			}
		}
	}
}

func syncProjects(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	for _, project := range gitlabContent.Projects {
		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(project.PathWithNameSpace)
		if actualNamespace != "" {

			// get expectedRoleBindings by retrieved Members
			expectedRoleBindings := map[string]bool{}

			// create or get ServiceAccount
			serviceAccountInfo, roleBindingName, err := k8sclient.CreateServiceAccountAndRoleBinding(project.PathWithNameSpace)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount. Err was: %s"), err)
			}
			expectedRoleBindings[roleBindingName] = true

			// configure project in gitlab for K8s integration
			go gitlabclient.SetupK8sIntegrationForGitlabProject(strconv.Itoa(project.Id), serviceAccountInfo.Namespace, serviceAccountInfo.Token)

			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)

			for _, member := range project.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				roleName := k8sclient.GetProjectRoleName(accessLevel)
				rbName := k8sclient.ConstructRoleBindingName(member.Username, roleName, actualNamespace)
				expectedRoleBindings[rbName] = true

				// make sure the project's expected rolebindings are present
				if !k8sRoleBindings[rbName] {
					k8sclient.CreateProjectRoleBinding(member.Username, project.PathWithNameSpace, accessLevel)
				}
			}

			// 2.1 Iterate all roleBindings and delete those which are not anymore present in gitlab
			// or through logic of this service
			for rb := range k8sRoleBindings {
				if !expectedRoleBindings[rb] && !cRaB.RoleBindings[rb] {
					k8sclient.DeleteProjectRoleBindingByName(rb, actualNamespace)
				}
			}

			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)
		} else {
			// create Namespace & RoleBinding
			k8sclient.CreateNamespace(project.PathWithNameSpace)
			serviceAccountInfo, _, err := k8sclient.CreateServiceAccountAndRoleBinding(project.PathWithNameSpace)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount. Err was: %s", err))
			}

			// configure project in gitlab for K8s integration
			go gitlabclient.SetupK8sIntegrationForGitlabProject(strconv.Itoa(project.Id), serviceAccountInfo.Namespace, serviceAccountInfo.Token)

			for _, member := range project.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				k8sclient.CreateProjectRoleBinding(member.Username, project.PathWithNameSpace, accessLevel)
			}

		}
	}
}

func StartRecurringSyncTimer() {
	log.Println("Starting Sync Timer...")
	ticker := time.NewTicker(time.Hour * 3)
	go func() {
		for range ticker.C {
			go PerformGlK8sSync()
		}
	}()
}

func debugSync() bool {
	return os.Getenv("ENABLE_GITLAB_SYNC_DEBUG") == "true"
}
<commit_msg>finished 1st version of graylog integration<commit_after>/*
	Copyright 2017 by Christian Hüning (christianhuening@googlemail.com).

	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at
		http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
*/

package usecases

import (
	"fmt"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/gitlabclient"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/k8sclient"
	"log"
	"os"
	"strconv"
	"sync"
	"time"
	"gitlab.informatik.haw-hamburg.de/icc/gl-k8s-integrator/graylog"
)

/*
What to fetch from k8s api
- Get all namespaces with gitlab-origin field (ns without that field won't be gitlab created)
- Get all rolebindings of these namespaces

What to get from gitlab
- get all groups
- get all projects
- get all users (private namespace)

Algo:
1. Delete all namespaces which are not in the gitlab Set
2. Iterate all gitlab namespaces
   if namespace is present in k8s set:
	2.1 Iterate all rolebindings
	2.2 Compare to rolebindings from k8s set by using the gitlab-origin field as key and
		2.2.1 Delete every rolebinding not present in the gitlab set
		2.2.1 Create every rolebinding not present in the k8s set
   else:
	2.1 Create namespace
		2.1.1 If namespace is present by name, but does not have a gitlab-origin label attached
		AND is not(!) labeled with 'gitlab-ignored' it get's labeled with its origin name.
		Otherwise the naming collision is solved by suffixing the name with a counter
	2.2 Create all rolebindings

 done

*/

// TODO : Cache Webhooks while Sync is running and execute them later!

func PerformGlK8sSync() {
	log.Println("Starting new Synchronization run!")
	log.Println("Getting Gitlab Contents...")
	gitlabContent, err := gitlabclient.GetFullGitlabContent()
	if check(err) {
		return
	}

	// 1. delete all Namespaces which are not in the gitlab set
	log.Println("Getting K8s Contents...")
	gitlabNamespacesInK8s := k8sclient.GetAllGitlabOriginNamesFromNamespacesWithOriginLabel()

	log.Println("Deleting all namespaces which are no longer in the gitlab namespace...")
	for _, originalName := range gitlabNamespacesInK8s {
		delete := true

		for _, user := range gitlabContent.Users {
			if originalName == user.Username {
				delete = false
				break
			}
		}

		if delete {
			for _, project := range gitlabContent.Projects {
				if originalName == project.PathWithNameSpace {
					delete = false
					break
				}
			}
		}

		if delete {
			for _, group := range gitlabContent.Groups {
				if originalName == group.FullPath {
					delete = false
					break
				}
			}
		}

		if delete {
			actualNs := k8sclient.DeleteNamespace(originalName)
			graylog.DeleteStream(actualNs)
		}
	}

	log.Println("Reading custom-rolebindings if any...")

	cRaB := ReadAndApplyCustomRolesAndBindings()

	var syncDoneWg sync.WaitGroup
	syncDoneWg.Add(3)

	log.Println("Syncing Gitlab Users...")
	go syncUsers(gitlabContent, cRaB, &syncDoneWg)

	log.Println("Syncing Gitlab Groups...")
	go syncGroups(gitlabContent, cRaB, &syncDoneWg)

	log.Println("Syncing Gitlab Projects...")
	go syncProjects(gitlabContent, cRaB, &syncDoneWg)

	syncDoneWg.Wait()
	log.Println("Finished Synchronization run.")
}

func syncUsers(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	for _, user := range gitlabContent.Users {
		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(user.Username)
		if actualNamespace != "" {

			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)
			roleName := k8sclient.GetGroupRoleName("Master")
			expectedGitlabRolebindingName := k8sclient.ConstructRoleBindingName(user.Username, roleName, actualNamespace)

			// 2.1 Iterate all roleBindings
			for rb := range k8sRoleBindings {
				if rb != expectedGitlabRolebindingName && !cRaB.RoleBindings[rb] {
					k8sclient.DeleteGroupRoleBindingByName(rb, actualNamespace)
					graylog.TakePermissionForStream(actualNamespace, user.Username)
				}
			}
			// make sure the project's role binding is present
			if !k8sRoleBindings[expectedGitlabRolebindingName] {
				k8sclient.CreateGroupRoleBinding(user.Username, user.Username, "Master")
				graylog.GrantPermissionForStream(actualNamespace, user.Username)
			}

			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)

		} else {
			// create Namespace & RoleBinding
			k8sclient.CreateNamespace(user.Username)
			k8sclient.CreateGroupRoleBinding(user.Username, user.Username, "Master")
			graylog.CreateStream(user.Username)
			graylog.GrantPermissionForStream(user.Username, user.Username)
		}
	}
}

func syncGroups(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	// same same for Groups
	for _, group := range gitlabContent.Groups {
		if group.FullPath == "kube-system" {
			continue
		} // ignore kube-system group

		if debugSync() {
			log.Println("Syncing: " + group.FullPath)
		}

		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(group.FullPath)
		if debugSync() {
			log.Println("ActualNamespace: " + actualNamespace)
		}
		if actualNamespace != "" {
			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)
			if debugSync() {
				log.Printf("Found %d rolebindings \n", len(k8sRoleBindings))
			}

			// get expectedRoleBindings by retrieved Members
			expectedRoleBindings := map[string]bool{}

			// create or get ServiceAccount
			_, roleBindingName, err := k8sclient.CreateServiceAccountAndRoleBinding(group.FullPath)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount for group %s. Err was: %s", group.FullPath, err))
			}
			expectedRoleBindings[roleBindingName] = true

			for _, member := range group.Members {
				if debugSync() {
					log.Println("Processing member " + member.Name)
				}
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				roleName := k8sclient.GetGroupRoleName(accessLevel)
				rbName := k8sclient.ConstructRoleBindingName(member.Username, roleName, actualNamespace)
				expectedRoleBindings[rbName] = true

				if debugSync() {
					log.Printf("AccessLevel: %s, roleName: %s, rbName: %s", accessLevel, roleName, rbName)
				}

				// make sure the groups's expected rolebindings are present
				if !k8sRoleBindings[rbName] {
					if debugSync() {
						log.Println("Creating RoleBinding " + rbName)
					}
					k8sclient.CreateGroupRoleBinding(member.Username, group.FullPath, accessLevel)
					graylog.GrantPermissionForStream(actualNamespace, member.Username)
				}
			}

			// 2.1 Iterate all roleBindings and delete those which are not anymore present in gitlab or in custom roles
			for rb := range k8sRoleBindings {
				if !expectedRoleBindings[rb] && !cRaB.RoleBindings[rb] {
					if debugSync() {
						log.Println("Deleting RoleBinding " + rb)
					}
					k8sclient.DeleteGroupRoleBindingByName(rb, actualNamespace)
				}
			}

			// TODO: Delete removed user roles from Graylog

			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)

		} else {
			// create Namespace & RoleBinding
			actualNs := k8sclient.CreateNamespace(group.FullPath)
			graylog.CreateStream(actualNs)
			_, _, err := k8sclient.CreateServiceAccountAndRoleBinding(group.FullPath)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount for group %s. Err was: %s", group.FullPath, err))
			}
			if debugSync() {
				log.Println("Creating Namespace for " + group.FullPath)
			}
			for _, member := range group.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				k8sclient.CreateGroupRoleBinding(member.Username, group.FullPath, accessLevel)
				graylog.GrantPermissionForStream(actualNs, member.Username)
			}
		}
	}
}

func syncProjects(gitlabContent *gitlabclient.GitlabContent, cRaB CustomRolesAndBindings, syncDoneWg *sync.WaitGroup) {
	defer syncDoneWg.Done()
	for _, project := range gitlabContent.Projects {
		actualNamespace := k8sclient.GetActualNameSpaceNameByGitlabName(project.PathWithNameSpace)
		if actualNamespace != "" {

			// get expectedRoleBindings by retrieved Members
			expectedRoleBindings := map[string]bool{}

			// create or get ServiceAccount
			serviceAccountInfo, roleBindingName, err := k8sclient.CreateServiceAccountAndRoleBinding(project.PathWithNameSpace)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount. Err was: %s"), err)
			}
			expectedRoleBindings[roleBindingName] = true

			// configure project in gitlab for K8s integration
			go gitlabclient.SetupK8sIntegrationForGitlabProject(strconv.Itoa(project.Id), serviceAccountInfo.Namespace, serviceAccountInfo.Token)

			// namespace is present, check rolebindings
			k8sRoleBindings := k8sclient.GetRoleBindingsByNamespace(actualNamespace)

			for _, member := range project.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				roleName := k8sclient.GetProjectRoleName(accessLevel)
				rbName := k8sclient.ConstructRoleBindingName(member.Username, roleName, actualNamespace)
				expectedRoleBindings[rbName] = true

				// make sure the project's expected rolebindings are present
				if !k8sRoleBindings[rbName] {
					k8sclient.CreateProjectRoleBinding(member.Username, project.PathWithNameSpace, accessLevel)
					graylog.GrantPermissionForStream(actualNamespace, member.Username)
				}
			}

			// 2.1 Iterate all roleBindings and delete those which are not anymore present in gitlab
			// or through logic of this service
			for rb := range k8sRoleBindings {
				if !expectedRoleBindings[rb] && !cRaB.RoleBindings[rb] {
					k8sclient.DeleteProjectRoleBindingByName(rb, actualNamespace)
				}
			}

			// TODO: Delete removed user roles from Graylog

			// finally check if namespace has CEPHSecretUser
			k8sclient.DeployCEPHSecretUser(actualNamespace)
		} else {
			// create Namespace & RoleBinding
			actualNs := k8sclient.CreateNamespace(project.PathWithNameSpace)
			graylog.CreateStream(actualNs)
			serviceAccountInfo, _, err := k8sclient.CreateServiceAccountAndRoleBinding(project.PathWithNameSpace)
			if err != nil {
				log.Fatalln(fmt.Sprintf("A fatal error occurred while creating a ServiceAccount. Err was: %s", err))
			}

			// configure project in gitlab for K8s integration
			go gitlabclient.SetupK8sIntegrationForGitlabProject(strconv.Itoa(project.Id), serviceAccountInfo.Namespace, serviceAccountInfo.Token)

			for _, member := range project.Members {
				accessLevel := gitlabclient.TranslateIntAccessLevels(member.AccessLevel)
				k8sclient.CreateProjectRoleBinding(member.Username, project.PathWithNameSpace, accessLevel)
				graylog.GrantPermissionForStream(actualNs, member.Username)
			}

		}
	}
}

func StartRecurringSyncTimer() {
	log.Println("Starting Sync Timer...")
	ticker := time.NewTicker(time.Hour * 3)
	go func() {
		for range ticker.C {
			go PerformGlK8sSync()
		}
	}()
}

func debugSync() bool {
	return os.Getenv("ENABLE_GITLAB_SYNC_DEBUG") == "true"
}
<|endoftext|>


<commit_before>// author: Jacky Boen

package main

import (
	"fmt"
	"github.com/veandco/go-sdl2/sdl"
	"os"
)

var winTitle string = "Go-SDL2 Render"
var winWidth, winHeight int = 800, 600

func run() int {
	var window *sdl.Window
	var renderer *sdl.Renderer
	var points []sdl.Point
	var rect sdl.Rect
	var rects []sdl.Rect

	window, err := sdl.CreateWindow(winTitle, sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		winWidth, winHeight, sdl.WINDOW_SHOWN)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create window: %s\n", err)
		return 1
	}
	defer window.Destroy()

	renderer, err = sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	if err != nil {
		fmt.Fprint(os.Stderr, "Failed to create renderer: %s\n", err)
		os.Exit(2)
	}
	renderer.Clear()
	defer renderer.Destroy()

	go func() {
		println("goroutine: A")
	}()

	sdl.CallQueue <- func() {
		renderer.SetDrawColor(255, 255, 255, 255)
		renderer.DrawPoint(150, 300)
		println("queue: A")
	}

	go func() {
		println("goroutine: B")
	}()

	sdl.CallQueue <- func() {
		renderer.SetDrawColor(0, 0, 255, 255)
		renderer.DrawLine(0, 0, 200, 200)
		println("queue: B")
	}

	go func() {
		println("goroutine: C")
	}()

	sdl.CallQueue <- func() {
		points = []sdl.Point{{0, 0}, {100, 300}, {100, 300}, {200, 0}}
		renderer.SetDrawColor(255, 255, 0, 255)
		renderer.DrawLines(points)
		println("queue: C")
	}

	go func() {
		println("goroutine: D")
	}()

	sdl.CallQueue <- func() {
		rect = sdl.Rect{300, 0, 200, 200}
		renderer.SetDrawColor(255, 0, 0, 255)
		renderer.DrawRect(&rect)
		println("queue: D")
	}

	go func() {
		println("goroutine: E")
	}()

	sdl.CallQueue <- func() {
		rects = []sdl.Rect{{400, 400, 100, 100}, {550, 350, 200, 200}}
		renderer.SetDrawColor(0, 255, 255, 255)
		renderer.DrawRects(rects)
		println("queue: E")
	}

	go func() {
		println("goroutine: F")
	}()

	sdl.CallQueue <- func() {
		rect = sdl.Rect{250, 250, 200, 200}
		renderer.SetDrawColor(0, 255, 0, 255)
		renderer.FillRect(&rect)
		println("queue: F")
	}

	go func() {
		println("goroutine: G")
	}()

	sdl.CallQueue <- func() {
		rects = []sdl.Rect{{500, 300, 100, 100}, {200, 300, 200, 200}}
		renderer.SetDrawColor(255, 0, 255, 255)
		renderer.FillRects(rects)
		println("queue: G")
	}

	go func() {
		println("goroutine: H")
	}()

	sdl.CallQueue <- func() {
		renderer.Present()
		println("queue: H")
	}

	sdl.Delay(2000)

	return 0
}

func main() {
	os.Exit(run())
}
<commit_msg>examples: render_queue: wrap every SDL2 calls in CallQueue<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/veandco/go-sdl2/sdl"
)

const (
	WindowTitle = "Go-SDL2 Render Queue"
	WindowWidth = 800
	WindowHeight = 600
)

func run() int {
	var window *sdl.Window
	var renderer *sdl.Renderer
	var points []sdl.Point
	var rect sdl.Rect
	var rects []sdl.Rect
	var err error

	sdl.CallQueue <- func() {
		window, err = sdl.CreateWindow(WindowTitle, sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED, WindowWidth, WindowHeight, sdl.WINDOW_SHOWN)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create window: %s\n", err)
		return 1
	}
	defer func() {
		sdl.CallQueue <- func() {
			window.Destroy()
		}
	}()

	sdl.CallQueue <- func() {
		renderer, err = sdl.CreateRenderer(window, -1, sdl.RENDERER_ACCELERATED)
	}
	if err != nil {
		fmt.Fprint(os.Stderr, "Failed to create renderer: %s\n", err)
		return 2
	}

	sdl.CallQueue <- func() {
		renderer.Clear()
	}
	defer func() {
		sdl.CallQueue <- func() {
			renderer.Destroy()
		}
	}()

	go func() {
		println("goroutine: A")
	}()

	sdl.CallQueue <- func() {
		renderer.SetDrawColor(255, 255, 255, 255)
		renderer.DrawPoint(150, 300)
		println("queue: A")
	}

	go func() {
		println("goroutine: B")
	}()

	sdl.CallQueue <- func() {
		renderer.SetDrawColor(0, 0, 255, 255)
		renderer.DrawLine(0, 0, 200, 200)
		println("queue: B")
	}

	go func() {
		println("goroutine: C")
	}()

	sdl.CallQueue <- func() {
		points = []sdl.Point{{0, 0}, {100, 300}, {100, 300}, {200, 0}}
		renderer.SetDrawColor(255, 255, 0, 255)
		renderer.DrawLines(points)
		println("queue: C")
	}

	go func() {
		println("goroutine: D")
	}()

	sdl.CallQueue <- func() {
		rect = sdl.Rect{300, 0, 200, 200}
		renderer.SetDrawColor(255, 0, 0, 255)
		renderer.DrawRect(&rect)
		println("queue: D")
	}

	go func() {
		println("goroutine: E")
	}()

	sdl.CallQueue <- func() {
		rects = []sdl.Rect{{400, 400, 100, 100}, {550, 350, 200, 200}}
		renderer.SetDrawColor(0, 255, 255, 255)
		renderer.DrawRects(rects)
		println("queue: E")
	}

	go func() {
		println("goroutine: F")
	}()

	sdl.CallQueue <- func() {
		rect = sdl.Rect{250, 250, 200, 200}
		renderer.SetDrawColor(0, 255, 0, 255)
		renderer.FillRect(&rect)
		println("queue: F")
	}

	go func() {
		println("goroutine: G")
	}()

	sdl.CallQueue <- func() {
		rects = []sdl.Rect{{500, 300, 100, 100}, {200, 300, 200, 200}}
		renderer.SetDrawColor(255, 0, 255, 255)
		renderer.FillRects(rects)
		println("queue: G")
	}

	go func() {
		println("goroutine: H")
	}()

	sdl.CallQueue <- func() {
		renderer.Present()
		println("queue: H")
	}

	sdl.CallQueue <- func() {
		sdl.Delay(2000)
	}

	return 0
}

func main() {
	os.Exit(run())
}
<|endoftext|>


<commit_before>package concurrent

//----------------------------------------------------------------------
// This file is part of Gospel.
// Copyright (C) 2011-2020 Bernd Fix
//
// Gospel is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Gospel is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Gospel.  If not, see <http://www.gnu.org/licenses/>.
//----------------------------------------------------------------------

import (
	"fmt"
)

// Error codes
var (
	ErrSignallerRetired = fmt.Errorf("Signaller retiered")
	ErrUnknownListener  = fmt.Errorf("Unknown signal listener")
)

// Signal can be any object (intrinsic or custom); it is the responsibility of
// the senders and receivers of signals to handle them accordingly.
type Signal interface{}

// ListenerOp codes
var (
	LISTENER_ADD  = 0
	LISTENER_DROP = 1
)

// ListenerOp represents an operation on the listener list:
type ListenerOp struct {
	ch chan Signal // listener channel
	op int         // 0=add, 1=delete
}

// Signaller manages signals send by senders for multiple listeners.
type Signaller struct {
	inCh   chan Signal          // channel for incoming signals
	outChs map[chan Signal]bool // channels for out-going signals

	cmdCh  chan *ListenerOp // internal channel to synchronize maintenance
	resCh  chan interface{} // channel for command results
	active bool             // is the signaller dispatching signals?
}

// NewSignaller instantiates a new signal manager:
func NewSignaller() *Signaller {
	// create a new instance and initialize it.
	s := &Signaller{
		inCh:   make(chan Signal),
		outChs: make(map[chan Signal]bool),
		cmdCh:  make(chan *ListenerOp),
		resCh:  make(chan interface{}),
		active: true,
	}
	// run the dispatch loop as long as the signaller is active.
	go func() {
		for s.active {
			select {
			case cmd := <-s.cmdCh:
				// handle listener list operation
				switch cmd.op {
				case LISTENER_ADD:
					// create a new listener channel
					out := make(chan Signal)
					s.outChs[out] = true
					s.resCh <- out
				case LISTENER_DROP:
					var err error
					if _, ok := s.outChs[cmd.ch]; !ok {
						err = ErrUnknownListener
					} else {
						delete(s.outChs, cmd.ch)
					}
					s.resCh <- err
				}
			case x := <-s.inCh:
				// dispatch received signals
				for out, active := range s.outChs {
					if active {
						out <- x
					}
				}
			default:
			}
		}
	}()
	return s
}

// Retire a signaller: This will terminate the dispatch loop for signals; no
// further send or listen operations are supported. A retired signaller cannot
// be re-activated.
func (s *Signaller) Retire() {
	s.active = false
}

// Send a signal to be dispatched to all listeners.
func (s *Signaller) Send(sig Signal) error {
	// check for active signaller
	if !s.active {
		return ErrSignallerRetired
	}
	s.inCh <- sig
	return nil
}

// Listen returns a channel to listen on
func (s *Signaller) Listen() chan Signal {
	// check for active signaller
	if !s.active {
		return nil
	}
	// trigger add operation.
	s.cmdCh <- &ListenerOp{op: LISTENER_ADD}
	return (<-s.resCh).(chan Signal)
}

// DropListener removes a listener from the list.
func (s *Signaller) Drop(out chan Signal) error {
	// check for active signaller
	if !s.active {
		return ErrSignallerRetired
	}
	// trigger delete operation
	s.cmdCh <- &ListenerOp{
		ch: out,
		op: LISTENER_DROP,
	}
	// handle error return for command.
	var err error
	res := <-s.resCh
	if res != nil {
		err = res.(error)
	}
	return err
}
<commit_msg>Removed non-blocking from select in dispatch loop.<commit_after>package concurrent

//----------------------------------------------------------------------
// This file is part of Gospel.
// Copyright (C) 2011-2020 Bernd Fix
//
// Gospel is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Gospel is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Gospel.  If not, see <http://www.gnu.org/licenses/>.
//----------------------------------------------------------------------

import (
	"fmt"
)

// Error codes
var (
	ErrSignallerRetired = fmt.Errorf("Signaller retiered")
	ErrUnknownListener  = fmt.Errorf("Unknown signal listener")
)

// Signal can be any object (intrinsic or custom); it is the responsibility of
// the senders and receivers of signals to handle them accordingly.
type Signal interface{}

// ListenerOp codes
var (
	LISTENER_ADD  = 0
	LISTENER_DROP = 1
)

// ListenerOp represents an operation on the listener list:
type ListenerOp struct {
	ch chan Signal // listener channel
	op int         // 0=add, 1=delete
}

// Signaller manages signals send by senders for multiple listeners.
type Signaller struct {
	inCh   chan Signal          // channel for incoming signals
	outChs map[chan Signal]bool // channels for out-going signals

	cmdCh  chan *ListenerOp // internal channel to synchronize maintenance
	resCh  chan interface{} // channel for command results
	active bool             // is the signaller dispatching signals?
}

// NewSignaller instantiates a new signal manager:
func NewSignaller() *Signaller {
	// create a new instance and initialize it.
	s := &Signaller{
		inCh:   make(chan Signal),
		outChs: make(map[chan Signal]bool),
		cmdCh:  make(chan *ListenerOp),
		resCh:  make(chan interface{}),
		active: true,
	}
	// run the dispatch loop as long as the signaller is active.
	go func() {
		for s.active {
			select {
			case cmd := <-s.cmdCh:
				// handle listener list operation
				switch cmd.op {
				case LISTENER_ADD:
					// create a new listener channel
					out := make(chan Signal)
					s.outChs[out] = true
					s.resCh <- out
				case LISTENER_DROP:
					var err error
					if _, ok := s.outChs[cmd.ch]; !ok {
						err = ErrUnknownListener
					} else {
						delete(s.outChs, cmd.ch)
					}
					s.resCh <- err
				}
			case x := <-s.inCh:
				// dispatch received signals
				for out, active := range s.outChs {
					if active {
						out <- x
					}
				}
			}
		}
	}()
	return s
}

// Retire a signaller: This will terminate the dispatch loop for signals; no
// further send or listen operations are supported. A retired signaller cannot
// be re-activated.
func (s *Signaller) Retire() {
	s.active = false
}

// Send a signal to be dispatched to all listeners.
func (s *Signaller) Send(sig Signal) error {
	// check for active signaller
	if !s.active {
		return ErrSignallerRetired
	}
	s.inCh <- sig
	return nil
}

// Listen returns a channel to listen on
func (s *Signaller) Listen() chan Signal {
	// check for active signaller
	if !s.active {
		return nil
	}
	// trigger add operation.
	s.cmdCh <- &ListenerOp{op: LISTENER_ADD}
	return (<-s.resCh).(chan Signal)
}

// DropListener removes a listener from the list.
func (s *Signaller) Drop(out chan Signal) error {
	// check for active signaller
	if !s.active {
		return ErrSignallerRetired
	}
	// trigger delete operation
	s.cmdCh <- &ListenerOp{
		ch: out,
		op: LISTENER_DROP,
	}
	// handle error return for command.
	var err error
	res := <-s.resCh
	if res != nil {
		err = res.(error)
	}
	return err
}
<|endoftext|>


<commit_before>package ws

import (
	"reflect"
	"unsafe"
)

// Cipher applies XOR cipher to the payload using mask.
// Offset is used to cipher chunked data (e.g. in io.Reader implementations).
//
// To convert masked data into unmasked data, or vice versa, the following
// algorithm is applied.  The same algorithm applies regardless of the
// direction of the translation, e.g., the same steps are applied to
// mask the data as to unmask the data.
func Cipher(payload, mask []byte, offset int) {
	if len(mask) != 4 {
		return
	}

	n := len(payload)
	if n < 8 {
		for i := 0; i < n; i++ {
			payload[i] ^= mask[(offset+i)%4]
		}
		return
	}

	// Calculate position in mask due to previously processed bytes number.
	mpos := offset % 4
	// Count number of bytes will processed one by one from the begining of payload.
	// Bitwise used to avoid additional if.
	ln := (4 - mpos) & 0x0b
	// Count number of bytes will processed one by one from the end of payload.
	// This is done to process payload by 8 bytes in each iteration of main loop.
	rn := (n - ln) % 8

	for i := 0; i < ln; i++ {
		payload[i] ^= mask[(mpos+i)%4]
	}
	for i := n - rn; i < n; i++ {
		payload[i] ^= mask[(mpos+i)%4]
	}

	ph := *(*reflect.SliceHeader)(unsafe.Pointer(&payload))
	mh := *(*reflect.SliceHeader)(unsafe.Pointer(&mask))

	m := *(*uint32)(unsafe.Pointer(mh.Data))
	m2 := uint64(m)<<32 | uint64(m)

	// Process the rest of bytes as uint64.
	for i := ln; i+8 <= n-rn; i += 8 {
		v := (*uint64)(unsafe.Pointer(ph.Data + uintptr(i)))
		*v = *v ^ m2
	}
}
<commit_msg>micro optimization<commit_after>package ws

import (
	"reflect"
	"unsafe"
)

var remain = [4]int{0, 3, 2, 1}

// Cipher applies XOR cipher to the payload using mask.
// Offset is used to cipher chunked data (e.g. in io.Reader implementations).
//
// To convert masked data into unmasked data, or vice versa, the following
// algorithm is applied.  The same algorithm applies regardless of the
// direction of the translation, e.g., the same steps are applied to
// mask the data as to unmask the data.
func Cipher(payload, mask []byte, offset int) {
	if len(mask) != 4 {
		return
	}

	n := len(payload)
	if n < 8 {
		for i := 0; i < n; i++ {
			payload[i] ^= mask[(offset+i)%4]
		}
		return
	}

	// Calculate position in mask due to previously processed bytes number.
	mpos := offset % 4
	// Count number of bytes will processed one by one from the begining of payload.
	ln := remain[mpos]
	// Count number of bytes will processed one by one from the end of payload.
	// This is done to process payload by 8 bytes in each iteration of main loop.
	rn := (n - ln) % 8

	for i := 0; i < ln; i++ {
		payload[i] ^= mask[(mpos+i)%4]
	}
	for i := n - rn; i < n; i++ {
		payload[i] ^= mask[(mpos+i)%4]
	}

	ph := *(*reflect.SliceHeader)(unsafe.Pointer(&payload))
	mh := *(*reflect.SliceHeader)(unsafe.Pointer(&mask))

	m := *(*uint32)(unsafe.Pointer(mh.Data))
	m2 := uint64(m)<<32 | uint64(m)

	// Process the rest of bytes as uint64.
	for i := ln; i+8 <= n-rn; i += 8 {
		v := (*uint64)(unsafe.Pointer(ph.Data + uintptr(i)))
		*v = *v ^ m2
	}
}
<|endoftext|>


<commit_before>// This package implements XXTEA encryption as defined in Needham and Wheeler's
// 1998 technical report, "Correction to XTEA."
package xxtea

// For details, see http://www.movable-type.co.uk/scripts/xxtea.pdf

import "strconv"

// The XXTEA block size in bytes.
const BlockSize = 8

// A Cipher is an instance of an XXTEA cipher using a particular key.
type Cipher struct {
	k [4]uint32
}

type KeySizeError int

func (k KeySizeError) Error() string {
	return "crypto/xtea: invalid key size " + strconv.Itoa(int(k))
}

// NewCipher creates and returns a new Cipher. The key argument should
// be the XXTEA key. XXTEA only supports 128 bit (16 byte) keys which
// are converted internally into 4 little-endian uint32 values.
func NewCipher(key []byte) (*Cipher, error) {
	k := len(key)
	switch k {
	default:
		return nil, KeySizeError(k)
	case 16:
		break
	}
	u := bytesToUint32(key)
	c := new(Cipher)
	copy(c.k[:], u)
	return c, nil
}

func (c *Cipher) BlockSize() int { return BlockSize }

func (c *Cipher) Encrypt(dst, src []byte) {
	v := bytesToUint32(src)
	c.BlockEncrypt(v)
	copy(dst, uint32ToBytes(v))
}

func (c *Cipher) Decrypt(dst, src []byte) {
	v := bytesToUint32(src)
	c.BlockDecrypt(v)
	copy(dst, uint32ToBytes(v))
}

const delta = 0x9e3779b9

// BlockEncrypt encrypts the []uint32 represtentation of a block,
// in-place.
func (c *Cipher) BlockEncrypt(v []uint32) {
	n := len(v)
	y := v[0]
	z := v[n-1]
	q := 6 + 52/n

	var sum uint32
	for q > 0 {
		q--
		sum += delta
		e := (sum >> 2) & 3
		var p int
		for p = 0; p < n-1; p++ {
			y = v[p+1]
			v[p] += ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
			z = v[p]
		}
		y = v[0]
		v[n-1] += ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
		z = v[n-1]
	}
}

// BlockDecrypt decrypts the []uint32 represtentation of a block,
// in-place.
func (c *Cipher) BlockDecrypt(v []uint32) {
	n := len(v)
	y := v[0]
	z := v[n-1]
	q := 6 + 52/n

	sum := uint32(q * delta)
	for sum != 0 {
		e := (sum >> 2) & 3
		var p int
		for p = n - 1; p > 0; p-- {
			z = v[p-1]
			v[p] -= ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
			y = v[p]
		}
		z = v[n-1]
		v[0] -= ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
		y = v[0]
		sum -= delta
	}
}
<commit_msg>Make NewCipher return a cipher.Block rather than concrete type; hide details<commit_after>// Package xxtea implements XXTEA encryption as defined in Needham and Wheeler's
// 1998 technical report, "Correction to XTEA."
package xxtea

// For details, see http://www.movable-type.co.uk/scripts/xxtea.pdf

import (
	"crypto/cipher"
	"strconv"
)

// The XXTEA block size in bytes.
const BlockSize = 8

// An xxteaCipher is an instance of an XXTEA cipher using a particular key.
type xxteaCipher struct {
	k [4]uint32
}

// KeySizeError may be returned by NewCipher.
type KeySizeError int

func (k KeySizeError) Error() string {
	return "crypto/xtea: invalid key size " + strconv.Itoa(int(k))
}

// NewCipher creates and returns a new cipher.Block. The key argument
// should be the XXTEA key. XXTEA only supports 128 bit (16 byte) keys
// which are converted internally into 4 little-endian uint32 values.
func NewCipher(key []byte) (cipher.Block, error) {
	k := len(key)
	switch k {
	default:
		return nil, KeySizeError(k)
	case 16:
		break
	}
	u := bytesToUint32(key)
	c := new(xxteaCipher)
	copy(c.k[:], u)
	return c, nil
}

func (c *xxteaCipher) BlockSize() int { return BlockSize }

func (c *xxteaCipher) Encrypt(dst, src []byte) {
	v := bytesToUint32(src)
	c.blockEncrypt(v)
	copy(dst, uint32ToBytes(v))
}

func (c *xxteaCipher) Decrypt(dst, src []byte) {
	v := bytesToUint32(src)
	c.blockDecrypt(v)
	copy(dst, uint32ToBytes(v))
}

const delta = 0x9e3779b9

// blockEncrypt encrypts the []uint32 represtentation of a block,
// in-place.
func (c *xxteaCipher) blockEncrypt(v []uint32) {
	n := len(v)
	y := v[0]
	z := v[n-1]
	q := 6 + 52/n

	var sum uint32
	for q > 0 {
		q--
		sum += delta
		e := (sum >> 2) & 3
		var p int
		for p = 0; p < n-1; p++ {
			y = v[p+1]
			v[p] += ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
			z = v[p]
		}
		y = v[0]
		v[n-1] += ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
		z = v[n-1]
	}
}

// blockDecrypt decrypts the []uint32 represtentation of a block,
// in-place.
func (c *xxteaCipher) blockDecrypt(v []uint32) {
	n := len(v)
	y := v[0]
	z := v[n-1]
	q := 6 + 52/n

	sum := uint32(q * delta)
	for sum != 0 {
		e := (sum >> 2) & 3
		var p int
		for p = n - 1; p > 0; p-- {
			z = v[p-1]
			v[p] -= ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
			y = v[p]
		}
		z = v[n-1]
		v[0] -= ((z>>5 ^ y<<2) + (y>>3 ^ z<<4)) ^ ((sum ^ y) + (c.k[uint32(p)&3^e] ^ z))
		y = v[0]
		sum -= delta
	}
}
<|endoftext|>


<commit_before>package clever

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

const debug = false

// Auth holds credentials for access to the API: basic auth with an API key or bearer auth with a token
type Auth struct {
	APIKey, Token string
}

// Clever wraps the Clever API at the specified URL e.g. "https://api.clever.com"
type Clever struct {
	Auth
	Url string
}

// BasicAuthTransport contains a user's auth credentials. Clever uses the OAuth Transport pattern with API keys.
type BasicAuthTransport struct {
	Username string
	Password string
}

// RoundTrip makes a request and returns the response
func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

// Client returns a new Client object for the specified BasicAuthTransport
func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// New returns a new Clever object to make requests with. URL must be a valid base url, e.g. "https://api.clever.com"
func New(auth Auth, url string) *Clever {
	return &Clever{auth, url}
}

// CleverError contains an error that occurred within the Clever API
type CleverError struct {
	Code    string
	Message string `json:"error"`
}

// Error returns a string representation of a CleverError
func (err *CleverError) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Error, err.Code)
}

// TooManyRequestsError indicates the number of requests has exceeded the rate limit
type TooManyRequestsError struct {
	Header http.Header
}

// TooManyRequestsError creates a TooManyRequestsError
func (err *TooManyRequestsError) Error() string {
	err_string := "Too Many Requests"
	for bucket_index, bucket_name := range err.Header[http.CanonicalHeaderKey("X-Ratelimit-Bucket")] {
		err_string += fmt.Sprintf("\nBucket: %s", bucket_name)
		for _, prop := range []string{"Remaining", "Limit", "Reset"} {
			headers_for_prop := err.Header[http.CanonicalHeaderKey("X-Ratelimit-"+prop)]
			if bucket_index < len(headers_for_prop) {
				err_string += fmt.Sprintf(", %s: %s", prop, headers_for_prop[bucket_index])
			}
		}
	}
	return err_string
}

// Paging contains information for paging a response
type Paging struct {
	Count   int
	Current int
	Total   int
}

// Link represents a stable link for querying the API
type Link struct {
	Rel string
	Uri string
}

// DistrictResp wraps the response given when the user queries for a District
type DistrictResp struct {
	District District `json:"data"`
	Links    []Link
	Uri      string
}

// District corresponds to the District resource in the Clever data schema: clever.com/schema
type District struct {
	Id   string
	Name string
}

// SchoolResp wraps the response given when the user queries for a School
type SchoolResp struct {
	Links  []Link
	School School `json:"data"`
	Uri    string
}

// School corresponds to the School resource in the Clever data schema: clever.com/schema
type School struct {
	Created      string
	District     string
	HighGrade    string `json:"high_grade"`
	Id           string
	LastModified string `json:"last_modified"`
	Location     Location
	LowGrade     string `json:"low_grade"`
	Name         string
	NcesId       string `json:"nces_id"`
	Phone        string
	SchoolNumber string `json:"school_number"`
	SisId        string `json:"sis_id"`
	StateId      string `json:"state_id"`
}

// TeacherResp wraps the response given when the user queries for a Teacher
type TeacherResp struct {
	Links   []Link
	Teacher Teacher `json:"data"`
	Uri     string
}

// Teacher corresponds to the Teacher resource in the Clever data schema: clever.com/schema
type Teacher struct {
	Created       string
	District      string
	Email         string
	Id            string
	LastModified  string `json:"last_modified"`
	Name          Name
	School        string
	SisId         string `json:"sis_id"`
	TeacherNumber string `json:"teacher_number"`
	Title         string
}

// StudentResp wraps the response given when the user queries for a Student
type StudentResp struct {
	Links   []Link
	Student Student `json:"data"`
	Uri     string
}

// Student corresponds to the Student resource in the Clever data schema: clever.com/schema
type Student struct {
	Created           string
	District          string
	Dob               string
	Email             string
	FrlStatus         string `json:"frl_status"`
	Gender            string
	Grade             string
	HispanicEthnicity string `json:"hispanic_ethnicity"`
	Id                string
	LastModified      string `json:"last_modified"`
	Location          Location
	Name              Name
	Race              string
	School            string
	SisId             string `json:"sis_id"`
	StateId           string `json:"state_id"`
	StudentNumber     string `json:"student_number"`
}

// SectionResp wraps the response given when the user queries for a Section
type SectionResp struct {
	Links   []Link
	Section Section `json:"data"`
	Uri     string
}

// Section corresponds to the Section resource in the Clever data schema: clever.com/schema
type Section struct {
	CourseName   string `json:"course_name"`
	CourseNumber string `json:"course_number"`
	Created      string
	District     string
	Grade        string
	Id           string `json:"id"`
	LastModified string `json:"last_modified"`
	Name         string
	School       string
	SisId        string `json:"sis_id"`
	Students     []string
	Subject      string
	Teacher      string
	Term
}

// Location represents a complete address for use with the Student and School resources
type Location struct {
	Address string
	City    string
	State   string
	Zip     string
}

// Name represents the full name of a Student or Teacher resource
type Name struct {
	First  string
	Middle string
	Last   string
}

// Term holds information about the duration of a school term
type Term struct {
	Name      string
	StartDate string `json:"start_date"`
	EndDate   string `json:"end_date"`
}

/*
Query makes a request to Clever given a Clever object, endpoint path, and parameters (pass in nil for no parameters).
*/
func (clever *Clever) Query(path string, params url.Values, resp interface{}) error {
	// Create request URI from Clever base, path, params
	uri := fmt.Sprintf("%s%s", clever.Url, path)
	if params != nil {
		uri = fmt.Sprintf("%s%s?%s", clever.Url, path, params.Encode())
	}

	// Ensure authentication is provided
	var client *http.Client
	req, _ := http.NewRequest("GET", uri, nil)
	if clever.Auth.Token != "" {
		t := &oauth.Transport{
			Token: &oauth.Token{AccessToken: clever.Auth.Token},
		}
		client = t.Client()
	} else if clever.Auth.APIKey != "" {
		t := &BasicAuthTransport{
			Username: clever.Auth.APIKey,
		}
		client = t.Client()
	} else {
		return fmt.Errorf("Must provide either API key or bearer token")
	}
	if debug {
		log.Printf("get { %v } -> {\n", uri)
	}
	r, err := client.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if debug {
		dump, _ := httputil.DumpResponse(r, true)
		log.Printf("response:\n")
		log.Printf("%v\n}\n", string(dump))
	}
	if r.StatusCode == 429 {
		return &TooManyRequestsError{r.Header}
	} else if r.StatusCode != 200 {
		var error CleverError
		json.NewDecoder(r.Body).Decode(&error)
		return &error
	}
	err = json.NewDecoder(r.Body).Decode(resp)
	return err
}

// PagedResult wraps a response. It allows for paged reading of a response in conjunction with QueryAll() and Next()
type PagedResult struct {
	clever         Clever
	nextPagePath   string
	lastData       []map[string]interface{}
	lastDataCursor int
	lastError      error
}

// QueryAll returns a PagedResult to allow for paged reading of large responses from the Clever API
func (clever *Clever) QueryAll(path string, params url.Values) PagedResult {
	paramString := ""
	if params != nil {
		paramString = "?" + params.Encode()
	}

	return PagedResult{clever: *clever, nextPagePath: path + paramString, lastDataCursor: -1}
}

/*
Next returns true if a PagedResult contains additional data and false if the cursor has reached
the end of the available data for this response.
*/
func (r *PagedResult) Next() bool {
	if r.lastDataCursor != -1 && r.lastDataCursor < len(r.lastData)-1 {
		r.lastDataCursor++
		return true
	}

	if r.nextPagePath == "" {
		return false
	}

	resp := &struct {
		Data   []map[string]interface{}
		Links  []Link
		Paging Paging
	}{}
	r.lastError = r.clever.Query(r.nextPagePath, nil, resp)
	if r.lastError != nil {
		return false
	}
	r.lastData = resp.Data
	r.lastDataCursor = 0
	r.nextPagePath = ""
	for _, link := range resp.Links {
		if link.Rel == "next" {
			r.nextPagePath = link.Uri
			break
		}
	}
	return len(r.lastData) > 0
}

// Scan parses the next page of results in a PagedResult r into result. Scan throws an error if r is invalid JSON.
func (r *PagedResult) Scan(result interface{}) error {
	data, err := json.Marshal(r.lastData[r.lastDataCursor]["data"])
	if err != nil {
		return err
	}
	return json.Unmarshal(data, result)
}

// Error returns the error in a response, if there is one
func (r *PagedResult) Error() error {
	return r.lastError
}
<commit_msg>gtfo underscores; underscores -> camelcase<commit_after>package clever

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

const debug = false

// Auth holds credentials for access to the API: basic auth with an API key or bearer auth with a token
type Auth struct {
	APIKey, Token string
}

// Clever wraps the Clever API at the specified URL e.g. "https://api.clever.com"
type Clever struct {
	Auth
	Url string
}

// BasicAuthTransport contains a user's auth credentials. Clever uses the OAuth Transport pattern with API keys.
type BasicAuthTransport struct {
	Username string
	Password string
}

// RoundTrip makes a request and returns the response
func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

// Client returns a new Client object for the specified BasicAuthTransport
func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// New returns a new Clever object to make requests with. URL must be a valid base url, e.g. "https://api.clever.com"
func New(auth Auth, url string) *Clever {
	return &Clever{auth, url}
}

// CleverError contains an error that occurred within the Clever API
type CleverError struct {
	Code    string
	Message string `json:"error"`
}

// Error returns a string representation of a CleverError
func (err *CleverError) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Error, err.Code)
}

// TooManyRequestsError indicates the number of requests has exceeded the rate limit
type TooManyRequestsError struct {
	Header http.Header
}

// TooManyRequestsError creates a TooManyRequestsError
func (err *TooManyRequestsError) Error() string {
	errString := "Too Many Requests"
	for bucketIndex, bucketName := range err.Header[http.CanonicalHeaderKey("X-Ratelimit-Bucket")] {
		errString += fmt.Sprintf("\nBucket: %s", bucketName)
		for _, prop := range []string{"Remaining", "Limit", "Reset"} {
			headersForProp := err.Header[http.CanonicalHeaderKey("X-Ratelimit-"+prop)]
			if bucketIndex < len(headersForProp) {
				errString += fmt.Sprintf(", %s: %s", prop, headers_for_prop[bucketIndex])
			}
		}
	}
	return errString
}

// Paging contains information for paging a response
type Paging struct {
	Count   int
	Current int
	Total   int
}

// Link represents a stable link for querying the API
type Link struct {
	Rel string
	Uri string
}

// DistrictResp wraps the response given when the user queries for a District
type DistrictResp struct {
	District District `json:"data"`
	Links    []Link
	Uri      string
}

// District corresponds to the District resource in the Clever data schema: clever.com/schema
type District struct {
	Id   string
	Name string
}

// SchoolResp wraps the response given when the user queries for a School
type SchoolResp struct {
	Links  []Link
	School School `json:"data"`
	Uri    string
}

// School corresponds to the School resource in the Clever data schema: clever.com/schema
type School struct {
	Created      string
	District     string
	HighGrade    string `json:"high_grade"`
	Id           string
	LastModified string `json:"last_modified"`
	Location     Location
	LowGrade     string `json:"low_grade"`
	Name         string
	NcesId       string `json:"nces_id"`
	Phone        string
	SchoolNumber string `json:"school_number"`
	SisId        string `json:"sis_id"`
	StateId      string `json:"state_id"`
}

// TeacherResp wraps the response given when the user queries for a Teacher
type TeacherResp struct {
	Links   []Link
	Teacher Teacher `json:"data"`
	Uri     string
}

// Teacher corresponds to the Teacher resource in the Clever data schema: clever.com/schema
type Teacher struct {
	Created       string
	District      string
	Email         string
	Id            string
	LastModified  string `json:"last_modified"`
	Name          Name
	School        string
	SisId         string `json:"sis_id"`
	TeacherNumber string `json:"teacher_number"`
	Title         string
}

// StudentResp wraps the response given when the user queries for a Student
type StudentResp struct {
	Links   []Link
	Student Student `json:"data"`
	Uri     string
}

// Student corresponds to the Student resource in the Clever data schema: clever.com/schema
type Student struct {
	Created           string
	District          string
	Dob               string
	Email             string
	FrlStatus         string `json:"frl_status"`
	Gender            string
	Grade             string
	HispanicEthnicity string `json:"hispanic_ethnicity"`
	Id                string
	LastModified      string `json:"last_modified"`
	Location          Location
	Name              Name
	Race              string
	School            string
	SisId             string `json:"sis_id"`
	StateId           string `json:"state_id"`
	StudentNumber     string `json:"student_number"`
}

// SectionResp wraps the response given when the user queries for a Section
type SectionResp struct {
	Links   []Link
	Section Section `json:"data"`
	Uri     string
}

// Section corresponds to the Section resource in the Clever data schema: clever.com/schema
type Section struct {
	CourseName   string `json:"course_name"`
	CourseNumber string `json:"course_number"`
	Created      string
	District     string
	Grade        string
	Id           string `json:"id"`
	LastModified string `json:"last_modified"`
	Name         string
	School       string
	SisId        string `json:"sis_id"`
	Students     []string
	Subject      string
	Teacher      string
	Term
}

// Location represents a complete address for use with the Student and School resources
type Location struct {
	Address string
	City    string
	State   string
	Zip     string
}

// Name represents the full name of a Student or Teacher resource
type Name struct {
	First  string
	Middle string
	Last   string
}

// Term holds information about the duration of a school term
type Term struct {
	Name      string
	StartDate string `json:"start_date"`
	EndDate   string `json:"end_date"`
}

/*
Query makes a request to Clever given a Clever object, endpoint path, and parameters (pass in nil for no parameters).
*/
func (clever *Clever) Query(path string, params url.Values, resp interface{}) error {
	// Create request URI from Clever base, path, params
	uri := fmt.Sprintf("%s%s", clever.Url, path)
	if params != nil {
		uri = fmt.Sprintf("%s%s?%s", clever.Url, path, params.Encode())
	}

	// Ensure authentication is provided
	var client *http.Client
	req, _ := http.NewRequest("GET", uri, nil)
	if clever.Auth.Token != "" {
		t := &oauth.Transport{
			Token: &oauth.Token{AccessToken: clever.Auth.Token},
		}
		client = t.Client()
	} else if clever.Auth.APIKey != "" {
		t := &BasicAuthTransport{
			Username: clever.Auth.APIKey,
		}
		client = t.Client()
	} else {
		return fmt.Errorf("Must provide either API key or bearer token")
	}
	if debug {
		log.Printf("get { %v } -> {\n", uri)
	}
	r, err := client.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if debug {
		dump, _ := httputil.DumpResponse(r, true)
		log.Printf("response:\n")
		log.Printf("%v\n}\n", string(dump))
	}
	if r.StatusCode == 429 {
		return &TooManyRequestsError{r.Header}
	} else if r.StatusCode != 200 {
		var error CleverError
		json.NewDecoder(r.Body).Decode(&error)
		return &error
	}
	err = json.NewDecoder(r.Body).Decode(resp)
	return err
}

// PagedResult wraps a response. It allows for paged reading of a response in conjunction with QueryAll() and Next()
type PagedResult struct {
	clever         Clever
	nextPagePath   string
	lastData       []map[string]interface{}
	lastDataCursor int
	lastError      error
}

// QueryAll returns a PagedResult to allow for paged reading of large responses from the Clever API
func (clever *Clever) QueryAll(path string, params url.Values) PagedResult {
	paramString := ""
	if params != nil {
		paramString = "?" + params.Encode()
	}

	return PagedResult{clever: *clever, nextPagePath: path + paramString, lastDataCursor: -1}
}

/*
Next returns true if a PagedResult contains additional data and false if the cursor has reached
the end of the available data for this response.
*/
func (r *PagedResult) Next() bool {
	if r.lastDataCursor != -1 && r.lastDataCursor < len(r.lastData)-1 {
		r.lastDataCursor++
		return true
	}

	if r.nextPagePath == "" {
		return false
	}

	resp := &struct {
		Data   []map[string]interface{}
		Links  []Link
		Paging Paging
	}{}
	r.lastError = r.clever.Query(r.nextPagePath, nil, resp)
	if r.lastError != nil {
		return false
	}
	r.lastData = resp.Data
	r.lastDataCursor = 0
	r.nextPagePath = ""
	for _, link := range resp.Links {
		if link.Rel == "next" {
			r.nextPagePath = link.Uri
			break
		}
	}
	return len(r.lastData) > 0
}

// Scan parses the next page of results in a PagedResult r into result. Scan throws an error if r is invalid JSON.
func (r *PagedResult) Scan(result interface{}) error {
	data, err := json.Marshal(r.lastData[r.lastDataCursor]["data"])
	if err != nil {
		return err
	}
	return json.Unmarshal(data, result)
}

// Error returns the error in a response, if there is one
func (r *PagedResult) Error() error {
	return r.lastError
}
<|endoftext|>


<commit_before>package clever

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

const debug = false

// API supports basic auth with an API key or bearer auth with a token
type Auth struct {
	APIKey, Token string
}

type Clever struct {
	Auth
	Url string
}

// Using the oauth Transport pattern with API keys
type BasicAuthTransport struct {
	Username string
	Password string
}

func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// Creates a new clever object to make requests with. URL must be a valid base url, e.g. "https://api.clever.com"
func New(auth Auth, url string) *Clever {
	return &Clever{auth, url}
}

type CleverError struct {
	Code    string
	Message string `json:"error"`
}

func (err *CleverError) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Error, err.Code)
}

type TooManyRequestsError struct {
	Header http.Header
}

func (err *TooManyRequestsError) Error() string {
	err_string := "Too Many Requests"
	err_props := []string{"Remaining", "Limit", "Reset"}
	for bucket_index, bucket_name := range err.Header["X-Ratelimit-Bucket"] {
		err_string += "\nBucket: " + bucket_name
		for _, prop := range err_props {
			key := "X-Ratelimit-" + prop
			if bucket_index < len(err.Header[key]) {
				err_string += ", " + prop + ": " + err.Header[key][bucket_index]
			}
		}
	}
	return err_string
}

type Paging struct {
	Count   int
	Current int
	Total   int
}

type Link struct {
	Rel string
	Uri string
}

type DistrictResp struct {
	District District `json:"data"`
	Links    []Link
	Uri      string
}

type District struct {
	Id   string
	Name string
}

type SchoolResp struct {
	Links  []Link
	School School `json:"data"`
	Uri    string
}

type School struct {
	Created      string
	District     string
	HighGrade    string `json:"high_grade"`
	Id           string
	LastModified string `json:"last_modified"`
	Location     Location
	LowGrade     string `json:"low_grade"`
	Name         string
	NcesId       string `json:"nces_id"`
	Phone        string
	SchoolNumber string `json:"school_number"`
	SisId        string `json:"sis_id"`
	StateId      string `json:"state_id"`
}

type TeacherResp struct {
	Links   []Link
	Teacher Teacher `json:"data"`
	Uri     string
}

type Teacher struct {
	Created       string
	District      string
	Email         string
	Id            string
	LastModified  string `json:"last_modified"`
	Name          Name
	School        string
	SisId         string `json:"sis_id"`
	TeacherNumber string `json:"teacher_number"`
	Title         string
}

type StudentResp struct {
	Links   []Link
	Student Student `json:"data"`
	Uri     string
}

type Student struct {
	Created           string
	District          string
	Dob               string
	Email             string
	FrlStatus         string `json:"frl_status"`
	Gender            string
	Grade             string
	HispanicEthnicity string `json:"hispanic_ethnicity"`
	Id                string
	LastModified      string `json:"last_modified"`
	Location          Location
	Name              Name
	Race              string
	School            string
	SisId             string `json:"sis_id"`
	StateId           string `json:"state_id"`
	StudentNumber     string `json:"student_number"`
}

type SectionResp struct {
	Links   []Link
	Section Section `json:"data"`
	Uri     string
}

type Section struct {
	CourseName   string `json:"course_name"`
	CourseNumber string `json:"course_number"`
	Created      string
	District     string
	Grade        string
	Id           string `json:"id"`
	LastModified string `json:"last_modified"`
	Name         string
	School       string
	SisId        string `json:"sis_id"`
	Students     []string
	Subject      string
	Teacher      string
	Term
}

type Location struct {
	Address string
	City    string
	State   string
	Zip     string
}

type Name struct {
	First  string
	Middle string
	Last   string
}

type Term struct {
	Name      string
	StartDate string `json:"start_date"`
	EndDate   string `json:"end_date"`
}

func (clever *Clever) Query(path string, params url.Values, resp interface{}) error {
	uri := fmt.Sprintf("%s%s", clever.Url, path)
	if params != nil {
		uri = fmt.Sprintf("%s%s?%s", clever.Url, path, params.Encode())
	}

	var client *http.Client
	req, _ := http.NewRequest("GET", uri, nil)
	if clever.Auth.Token != "" {
		t := &oauth.Transport{
			Token: &oauth.Token{AccessToken: clever.Auth.Token},
		}
		client = t.Client()
	} else if clever.Auth.APIKey != "" {
		t := &BasicAuthTransport{
			Username: clever.Auth.APIKey,
		}
		client = t.Client()
	} else {
		return fmt.Errorf("Must provide either API key or bearer token")
	}
	if debug {
		log.Printf("get { %v } -> {\n", uri)
	}
	r, err := client.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if debug {
		dump, _ := httputil.DumpResponse(r, true)
		log.Printf("response:\n")
		log.Printf("%v\n}\n", string(dump))
	}
	if r.StatusCode == 429 {
		error := TooManyRequestsError{r.Header}
		return &error
	} else if r.StatusCode != 200 {
		var error CleverError
		json.NewDecoder(r.Body).Decode(&error)
		return &error
	}
	err = json.NewDecoder(r.Body).Decode(resp)
	return err
}

type PagedResult struct {
	clever         Clever
	nextPagePath   string
	lastData       []map[string]interface{}
	lastDataCursor int
	lastError      error
}

func (clever *Clever) QueryAll(path string, params url.Values) PagedResult {
	paramString := ""
	if params != nil {
		paramString = "?" + params.Encode()
	}

	return PagedResult{clever: *clever, nextPagePath: path + paramString, lastDataCursor: -1}
}

func (r *PagedResult) Next() bool {
	if r.lastDataCursor != -1 && r.lastDataCursor < len(r.lastData)-1 {
		r.lastDataCursor++
		return true
	}

	if r.nextPagePath == "" {
		return false
	}

	resp := &struct {
		Data   []map[string]interface{}
		Links  []Link
		Paging Paging
	}{}
	r.lastError = r.clever.Query(r.nextPagePath, nil, resp)
	if r.lastError != nil {
		return false
	}
	r.lastData = resp.Data
	r.lastDataCursor = 0
	r.nextPagePath = ""
	for _, link := range resp.Links {
		if link.Rel == "next" {
			r.nextPagePath = link.Uri
			break
		}
	}
	return len(r.lastData) > 0
}

func (r *PagedResult) Scan(result interface{}) error {
	data, err := json.Marshal(r.lastData[r.lastDataCursor]["data"])
	if err != nil {
		return err
	}
	return json.Unmarshal(data, result)
}

func (r *PagedResult) Error() error {
	return r.lastError
}
<commit_msg>Return struct literal without declaring first<commit_after>package clever

import (
	"code.google.com/p/goauth2/oauth"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/http/httputil"
	"net/url"
)

const debug = false

// API supports basic auth with an API key or bearer auth with a token
type Auth struct {
	APIKey, Token string
}

type Clever struct {
	Auth
	Url string
}

// Using the oauth Transport pattern with API keys
type BasicAuthTransport struct {
	Username string
	Password string
}

func (bat BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req.Header.Set("Authorization", fmt.Sprintf("Basic %s",
		base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s",
			bat.Username, bat.Password)))))
	return http.DefaultTransport.RoundTrip(req)
}

func (bat *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: bat}
}

// Creates a new clever object to make requests with. URL must be a valid base url, e.g. "https://api.clever.com"
func New(auth Auth, url string) *Clever {
	return &Clever{auth, url}
}

type CleverError struct {
	Code    string
	Message string `json:"error"`
}

func (err *CleverError) Error() string {
	if err.Code == "" {
		return err.Message
	}
	return fmt.Sprintf("%s (%s)", err.Error, err.Code)
}

type TooManyRequestsError struct {
	Header http.Header
}

func (err *TooManyRequestsError) Error() string {
	err_string := "Too Many Requests"
	err_props := []string{"Remaining", "Limit", "Reset"}
	for bucket_index, bucket_name := range err.Header["X-Ratelimit-Bucket"] {
		err_string += "\nBucket: " + bucket_name
		for _, prop := range err_props {
			key := "X-Ratelimit-" + prop
			if bucket_index < len(err.Header[key]) {
				err_string += ", " + prop + ": " + err.Header[key][bucket_index]
			}
		}
	}
	return err_string
}

type Paging struct {
	Count   int
	Current int
	Total   int
}

type Link struct {
	Rel string
	Uri string
}

type DistrictResp struct {
	District District `json:"data"`
	Links    []Link
	Uri      string
}

type District struct {
	Id   string
	Name string
}

type SchoolResp struct {
	Links  []Link
	School School `json:"data"`
	Uri    string
}

type School struct {
	Created      string
	District     string
	HighGrade    string `json:"high_grade"`
	Id           string
	LastModified string `json:"last_modified"`
	Location     Location
	LowGrade     string `json:"low_grade"`
	Name         string
	NcesId       string `json:"nces_id"`
	Phone        string
	SchoolNumber string `json:"school_number"`
	SisId        string `json:"sis_id"`
	StateId      string `json:"state_id"`
}

type TeacherResp struct {
	Links   []Link
	Teacher Teacher `json:"data"`
	Uri     string
}

type Teacher struct {
	Created       string
	District      string
	Email         string
	Id            string
	LastModified  string `json:"last_modified"`
	Name          Name
	School        string
	SisId         string `json:"sis_id"`
	TeacherNumber string `json:"teacher_number"`
	Title         string
}

type StudentResp struct {
	Links   []Link
	Student Student `json:"data"`
	Uri     string
}

type Student struct {
	Created           string
	District          string
	Dob               string
	Email             string
	FrlStatus         string `json:"frl_status"`
	Gender            string
	Grade             string
	HispanicEthnicity string `json:"hispanic_ethnicity"`
	Id                string
	LastModified      string `json:"last_modified"`
	Location          Location
	Name              Name
	Race              string
	School            string
	SisId             string `json:"sis_id"`
	StateId           string `json:"state_id"`
	StudentNumber     string `json:"student_number"`
}

type SectionResp struct {
	Links   []Link
	Section Section `json:"data"`
	Uri     string
}

type Section struct {
	CourseName   string `json:"course_name"`
	CourseNumber string `json:"course_number"`
	Created      string
	District     string
	Grade        string
	Id           string `json:"id"`
	LastModified string `json:"last_modified"`
	Name         string
	School       string
	SisId        string `json:"sis_id"`
	Students     []string
	Subject      string
	Teacher      string
	Term
}

type Location struct {
	Address string
	City    string
	State   string
	Zip     string
}

type Name struct {
	First  string
	Middle string
	Last   string
}

type Term struct {
	Name      string
	StartDate string `json:"start_date"`
	EndDate   string `json:"end_date"`
}

func (clever *Clever) Query(path string, params url.Values, resp interface{}) error {
	uri := fmt.Sprintf("%s%s", clever.Url, path)
	if params != nil {
		uri = fmt.Sprintf("%s%s?%s", clever.Url, path, params.Encode())
	}

	var client *http.Client
	req, _ := http.NewRequest("GET", uri, nil)
	if clever.Auth.Token != "" {
		t := &oauth.Transport{
			Token: &oauth.Token{AccessToken: clever.Auth.Token},
		}
		client = t.Client()
	} else if clever.Auth.APIKey != "" {
		t := &BasicAuthTransport{
			Username: clever.Auth.APIKey,
		}
		client = t.Client()
	} else {
		return fmt.Errorf("Must provide either API key or bearer token")
	}
	if debug {
		log.Printf("get { %v } -> {\n", uri)
	}
	r, err := client.Do(req)
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if debug {
		dump, _ := httputil.DumpResponse(r, true)
		log.Printf("response:\n")
		log.Printf("%v\n}\n", string(dump))
	}
	if r.StatusCode == 429 {
		return &TooManyRequestsError{r.Header}
	} else if r.StatusCode != 200 {
		var error CleverError
		json.NewDecoder(r.Body).Decode(&error)
		return &error
	}
	err = json.NewDecoder(r.Body).Decode(resp)
	return err
}

type PagedResult struct {
	clever         Clever
	nextPagePath   string
	lastData       []map[string]interface{}
	lastDataCursor int
	lastError      error
}

func (clever *Clever) QueryAll(path string, params url.Values) PagedResult {
	paramString := ""
	if params != nil {
		paramString = "?" + params.Encode()
	}

	return PagedResult{clever: *clever, nextPagePath: path + paramString, lastDataCursor: -1}
}

func (r *PagedResult) Next() bool {
	if r.lastDataCursor != -1 && r.lastDataCursor < len(r.lastData)-1 {
		r.lastDataCursor++
		return true
	}

	if r.nextPagePath == "" {
		return false
	}

	resp := &struct {
		Data   []map[string]interface{}
		Links  []Link
		Paging Paging
	}{}
	r.lastError = r.clever.Query(r.nextPagePath, nil, resp)
	if r.lastError != nil {
		return false
	}
	r.lastData = resp.Data
	r.lastDataCursor = 0
	r.nextPagePath = ""
	for _, link := range resp.Links {
		if link.Rel == "next" {
			r.nextPagePath = link.Uri
			break
		}
	}
	return len(r.lastData) > 0
}

func (r *PagedResult) Scan(result interface{}) error {
	data, err := json.Marshal(r.lastData[r.lastDataCursor]["data"])
	if err != nil {
		return err
	}
	return json.Unmarshal(data, result)
}

func (r *PagedResult) Error() error {
	return r.lastError
}
<|endoftext|>