Prepare for release (#14)

Author: Hugo Correia

README.md

@@ -1,19 +1,59 @@
 # dynamodbcopy
+
 [![Build Status](https://travis-ci.org/uniplaces/dynamodbcopy.svg?branch=master)](https://travis-ci.org/uniplaces/dynamodbcopy)
 [![Go Report Card](https://goreportcard.com/badge/github.com/uniplaces/dynamodbcopy)](https://goreportcard.com/report/github.com/uniplaces/dynamodbcopy)
 [![codecov](https://codecov.io/gh/uniplaces/dynamodbcopy/branch/master/graph/badge.svg)](https://codecov.io/gh/uniplaces/dynamodbcopy)
 [![GoDoc](https://godoc.org/github.com/uniplaces/dynamodbcopy?status.svg)](https://godoc.org/github.com/uniplaces/dynamodbcopy)
 [![License](http://img.shields.io/:license-apache-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0.html)
 
-## Development
-To build and run this cmd, you'll need go (1.11.x) with mod support enabled `export GO111MODULE=on`
-
-### Running
-To run:
-```
-go run cmd/main.go
-```
-Alternatively, you can easily build the binary by:
-```
-go build -o dynamodbcopy cmd/main.go
-```
+Dynamodbcopy is a cli tool wrapper around the [aws-sdk](https://github.com/aws/aws-sdk-go) that allows you to copy information from one dynamodb table to another.
+
+## Main Features
+
+- Provides a CLI to easily copy dynamodb records from one place to another
+- Allows you to set read and write capacity units for the source and target table
+- Integrates with [aws-sdk](https://github.com/aws/aws-sdk-go), sharing it's credentials
+- Allows you to parameterize the source and target table with specific roles, enabling you to perform cross-account copies
+- Stores current provisioning values before performing a copy, restoring the inital values at the end of the copy or if any error occurs during the copy.
+
+## Usage
+
+> Use "dynamodbcopy [command] --help" for more information about a command.
+
+## Installing
+
+Use go get to retrieve `dynamodbcopy` to add it to your GOPATH workspace, or project's Go module dependencies.
+
+> go get github.com/uniplaces/dynamodbcopy/cmd/dynamodbcopy
+
+To update run with `-u`
+
+> go get -u github.com/uniplaces/dynamodbcopy/cmd/dynamodbcopy
+
+### Go Modules
+
+If you are using Go modules, your go get will default to the latest tagged version. To get a specific release version of the `dynamodbcopy` use `@<tag>` in your go get command.
+
+> go get github.com/uniplaces/dynamodbcopy/cmd/[email protected]
+
+To get the latest repository change use `@latest` tag.
+
+> go get github.com/uniplaces/dynamodbcopy/cmd/dynamodbcopy@latest
+
+## Opening Issues
+
+If you encounter a bug, please start by searching the existing issues and see if others are also experiencing the issue before opening a new one. Please include the version for `dynamodbcopy` and Go that you are using. Please also include reproduction case when appropriate.
+
+## Contributing
+
+Please feel free to make suggestions, create issues, fork the repository and send pull requests!
+
+## Licence
+
+Copyright 2018 UNIPLACES
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

cmd/main.go cmd/dynamodbcopy/main.go

@@ -1,12 +1,14 @@
 package dynamodbcopy
 
+// Config encapsulates the values nedeed for the command
 type Config struct {
 	readCapacityUnits  int64
 	writeCapacityUnits int64
 	readWorkers        int
 	writeWorkers       int
 }
 
+// NewConfig creates a new Config to store the parameters user defined parameters
 func NewConfig(readUnits, writeUnits, readWorkers, writeWorkers int) Config {
 	return Config{
 		readCapacityUnits:  int64(readUnits),
@@ -16,6 +18,8 @@ func NewConfig(readUnits, writeUnits, readWorkers, writeWorkers int) Config {
 	}
 }
 
+// Provisioning calculates a new Provisioning value based on the passed argument and the current Config (receiver).
+// The returned Provisioning value will have the higher values for read and write capacity units of the 2
 func (c Config) Provisioning(current Provisioning) Provisioning {
 	src := current.Source
 	if src != nil && c.readCapacityUnits > src.Read {
@@ -30,6 +34,7 @@ func (c Config) Provisioning(current Provisioning) Provisioning {
 	return Provisioning{Source: src, Target: trg}
 }
 
+// Workers returns the Config read and write worker count
 func (c Config) Workers() (int, int) {
 	return c.readWorkers, c.writeWorkers
 }

config.go

@@ -5,29 +5,33 @@ import (
 	"sync"
 )
 
+// Copier is the interface that allows you to copy records from the source to target table
 type Copier interface {
 	Copy(readers, writers int) error
 }
 
 type copyService struct {
-	srcTable DynamoDBService
-	trgTable DynamoDBService
-	chans    CopierChans
-	logger   Logger
+	srcTable   DynamoDBService
+	trgTable   DynamoDBService
+	copierChan CopierChan
+	logger     Logger
 }
 
-func NewCopier(srcTableService, trgTableService DynamoDBService, chans CopierChans, logger Logger) Copier {
+// NewCopier returns a new Copier to copy records
+func NewCopier(srcTableService, trgTableService DynamoDBService, copierChan CopierChan, logger Logger) Copier {
 	return copyService{
-		srcTable: srcTableService,
-		trgTable: trgTableService,
-		chans:    chans,
-		logger:   logger,
+		srcTable:   srcTableService,
+		trgTable:   trgTableService,
+		copierChan: copierChan,
+		logger:     logger,
 	}
 }
 
+// Copy will copy all records from the source to target table.
+// This method will create a worker pool according to the number of readers and writes that are passed as argument
 func (service copyService) Copy(readers, writers int) error {
 	service.logger.Printf("copying table with %d readers and %d writers", readers, writers)
-	itemsChan, errChan := service.chans.Items, service.chans.Errors
+	itemsChan, errChan := service.copierChan.Items, service.copierChan.Errors
 
 	wgReaders := &sync.WaitGroup{}
 	wgReaders.Add(readers)
@@ -93,14 +97,15 @@ func (service copyService) write(wg *sync.WaitGroup, itemsChan <-chan []DynamoDB
 	service.logger.Printf("writer wrote a total of %d items", totalWritten)
 }
 
-// CopierChans encapsulates the chan that are used by the copier
-type CopierChans struct {
+// CopierChan encapsulates the value and error channel used by the copier
+type CopierChan struct {
 	Items  chan []DynamoDBItem
 	Errors chan error
 }
 
-func NewCopierChans(itemsChanSize int) CopierChans {
-	return CopierChans{
+// NewCopierChan creates a new CopierChan with a buffered chan []DynamoDBItem of itemsChanSize
+func NewCopierChan(itemsChanSize int) CopierChan {
+	return CopierChan{
 		Items:  make(chan []DynamoDBItem, itemsChanSize),
 		Errors: make(chan error),
 	}

copier.go

@@ -23,14 +23,14 @@ func TestCopy(t *testing.T) {
 
 	testCases := []struct {
 		subTestName   string
-		mocker        func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans)
+		mocker        func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan)
 		totalReaders  int
 		totalWriters  int
 		expectedError error
 	}{
 		{
 			"ScanError",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 1, 0, readChan).Return(scanError).Once()
 			},
@@ -40,7 +40,7 @@ func TestCopy(t *testing.T) {
 		},
 		{
 			"BatchWriteError",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 1, 0, readChan).Return(nil).Once()
 
@@ -54,7 +54,7 @@ func TestCopy(t *testing.T) {
 		},
 		{
 			"Success",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 1, 0, readChan).Return(nil).Once()
 
@@ -68,7 +68,7 @@ func TestCopy(t *testing.T) {
 		},
 		{
 			"MultipleWorkers",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 3, 0, readChan).Return(nil).Once()
 				src.On("Scan", 3, 1, readChan).Return(nil).Once()
@@ -92,7 +92,7 @@ func TestCopy(t *testing.T) {
 		},
 		{
 			"ReadPanic",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 1, 0, readChan).Run(func(args mock.Arguments) {
 					panic("read panic")
@@ -104,7 +104,7 @@ func TestCopy(t *testing.T) {
 		},
 		{
 			"WritePanic",
-			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChans) {
+			func(src, trg *mocks.DynamoDBService, chans *dynamodbcopy.CopierChan) {
 				var readChan chan<- []dynamodbcopy.DynamoDBItem = chans.Items
 				src.On("Scan", 1, 0, readChan).Return(nil).Once()
 
@@ -127,7 +127,7 @@ func TestCopy(t *testing.T) {
 				src := &mocks.DynamoDBService{}
 				trg := &mocks.DynamoDBService{}
 
-				copierChans := dynamodbcopy.NewCopierChans(testCase.totalWriters)
+				copierChans := dynamodbcopy.NewCopierChan(testCase.totalWriters)
 
 				testCase.mocker(src, trg, &copierChans)
 

copier_test.go

@@ -0,0 +1,2 @@
+// Package dynamodbcopy provides a simple interface for you to copy info across dynamodb tables
+package dynamodbcopy

doc.go

@@ -20,13 +20,39 @@ const (
 	errCodeThrottlingException = "ThrottlingException"
 )
 
-// DynamoDBAPI just a wrapper over aws-sdk dynamodbiface.DynamoDBAPI interface for mocking purposes
-type DynamoDBAPI interface {
+// DynamoDBClient is a wrapper interface over aws-sdk dynamodbiface.DynamoDBClient for mocking purposes
+type DynamoDBClient interface {
 	dynamodbiface.DynamoDBAPI
 }
 
+// NewDynamoClient creates a DynamoDB client wrapper around the AWS-SDK with a predefined Session.
+// By default, it creates a new Session with SharedConfigEnable,
+// so you can use AWS SDK's environment variables and AWS credentials to connect to DynamoDB.
+//
+// The provided ARN role allows you to configure the Session to assume a specific IAM Role.
+//
+// If an empty string is provided, it will create a new session with SharedConfigEnable
+// Please refer to https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html for more
+// information on how you can set up the SDK
+func NewDynamoClient(roleArn string) DynamoDBClient {
+	options := session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+	}
+
+	currentSession := session.Must(session.NewSessionWithOptions(options))
+	if roleArn != "" {
+		roleCredentials := stscreds.NewCredentials(currentSession, roleArn)
+
+		return dynamodb.New(currentSession, &aws.Config{Credentials: roleCredentials})
+	}
+
+	return dynamodb.New(currentSession)
+}
+
+// DynamoDBItem type to abstract a DynamoDB item
 type DynamoDBItem map[string]*dynamodb.AttributeValue
 
+// DynamoDBService interface provides methods to call the aws sdk
 type DynamoDBService interface {
 	DescribeTable() (*dynamodb.TableDescription, error)
 	UpdateCapacity(capacity Capacity) error
@@ -37,43 +63,31 @@ type DynamoDBService interface {
 
 type dynamoDBSerivce struct {
 	tableName string
-	api       DynamoDBAPI
+	client    DynamoDBClient
 	sleep     Sleeper
 	logger    Logger
 }
 
-func NewDynamoDBAPI(roleArn string) DynamoDBAPI {
-	options := session.Options{
-		SharedConfigState: session.SharedConfigEnable,
-	}
-
-	currentSession := session.Must(session.NewSessionWithOptions(options))
-	if roleArn != "" {
-		roleCredentials := stscreds.NewCredentials(currentSession, roleArn)
-
-		return dynamodb.New(currentSession, &aws.Config{Credentials: roleCredentials})
-	}
-
-	return dynamodb.New(currentSession)
-}
-
-func NewDynamoDBService(tableName string, api DynamoDBAPI, sleepFn Sleeper, logger Logger) DynamoDBService {
-	return dynamoDBSerivce{tableName, api, sleepFn, logger}
+// NewDynamoDBService creates new service for a given DynamoDB table with a previously configured DynamoDB client
+func NewDynamoDBService(tableName string, client DynamoDBClient, sleepFn Sleeper, logger Logger) DynamoDBService {
+	return dynamoDBSerivce{tableName, client, sleepFn, logger}
 }
 
+// DescribeTable returns the current table metadata for the DynamoDB table
 func (db dynamoDBSerivce) DescribeTable() (*dynamodb.TableDescription, error) {
 	input := &dynamodb.DescribeTableInput{
 		TableName: aws.String(db.tableName),
 	}
 
-	output, err := db.api.DescribeTable(input)
+	output, err := db.client.DescribeTable(input)
 	if err != nil {
 		return nil, fmt.Errorf("unable to describe table %s: %s", db.tableName, err)
 	}
 
 	return output.Table, nil
 }
 
+// UpdateCapacity sets the tables read and write capacity, waiting for the table to be ready for processing
 func (db dynamoDBSerivce) UpdateCapacity(capacity Capacity) error {
 	read := capacity.Read
 	write := capacity.Write
@@ -95,14 +109,21 @@ func (db dynamoDBSerivce) UpdateCapacity(capacity Capacity) error {
 	}
 
 	db.logger.Printf("updating %s with read: %d, write: %d", db.tableName, read, write)
-	_, err := db.api.UpdateTable(input)
+	_, err := db.client.UpdateTable(input)
 	if err != nil {
 		return fmt.Errorf("unable to update table %s: %s", db.tableName, err)
 	}
 
 	return db.WaitForReadyTable()
 }
 
+// BatchWrite writes the given DynamoDBItem slice into the DynamoDB table.
+//
+// The given items will be written in groups of 25 each.
+//
+// This method will retry:
+// 	1 - if there are any any unprocessed items when performing the BatchWrite
+// 	2 - if there is a Provisioning or Throttling aws error (tries for a max time of 3 minutes)
 func (db dynamoDBSerivce) BatchWrite(items []DynamoDBItem) error {
 	db.logger.Printf("writing batch of %d to %s", len(items), db.tableName)
 	if len(items) == 0 {
@@ -142,7 +163,7 @@ func (db dynamoDBSerivce) batchWriteItem(requests []*dynamodb.WriteRequest) erro
 				},
 			}
 
-			output, err := db.api.BatchWriteItem(batchInput)
+			output, err := db.client.BatchWriteItem(batchInput)
 			if err == nil {
 				writeRequests = output.UnprocessedItems[tableName]
 
@@ -178,6 +199,7 @@ func (db dynamoDBSerivce) batchWriteItem(requests []*dynamodb.WriteRequest) erro
 	return nil
 }
 
+// WaitForReadyTable will wait for the table status to be active (waits for 3 minutes)
 func (db dynamoDBSerivce) WaitForReadyTable() error {
 	return db.retry(func(attempt, elapsed int) (bool, error) {
 		description, err := db.DescribeTable()
@@ -207,6 +229,8 @@ func (db dynamoDBSerivce) retry(handler func(attempt, elapsed int) (bool, error)
 	return fmt.Errorf("waited for too long (%d ms) to perform operation on %s table", elapsed, db.tableName)
 }
 
+// Scan allows you to perform a parallel scan over the table, writing the scanned items into the provided itemsChan
+// If totalSegments is equal to 1, it will perform a sequential scan.
 func (db dynamoDBSerivce) Scan(totalSegments, segment int, itemsChan chan<- []DynamoDBItem) error {
 	if totalSegments == 0 {
 		return errors.New("totalSegments has to be greater than 0")
@@ -235,7 +259,7 @@ func (db dynamoDBSerivce) Scan(totalSegments, segment int, itemsChan chan<- []Dy
 		return !b
 	}
 
-	if err := db.api.ScanPages(&input, pagerFn); err != nil {
+	if err := db.client.ScanPages(&input, pagerFn); err != nil {
 		return fmt.Errorf("unable to scan table %s: %s", db.tableName, err)
 	}