Merge pull request #201 from stefanbenten/sb/put-full

Add second S3 Handler for Non-Multipart Uploads

.gitignore

@@ -5,6 +5,7 @@ src/
 bin/
 *.pyc
 *.egg-info/
+.idea/
 
 .tmp
 .vagrant

README.md

@@ -126,6 +126,7 @@ provider | which storage provider to use | (s3, grdrive or local) |
 aws-access-key | aws access key | | AWS_ACCESS_KEY
 aws-secret-key | aws access key | | AWS_SECRET_KEY
 bucket | aws bucket | | BUCKET
+s3-no-multipart | disables s3 multipart upload | false | |
 basedir | path storage for local/gdrive provider| |
 gdrive-client-json-filepath | path to oauth client json config for gdrive provider| |
 gdrive-local-config-path | path to store local transfer.sh config cache for gdrive provider| |

cmd/cmd.go

@@ -2,15 +2,13 @@ package cmd
 
 import (
 	"fmt"
-
+	"log"
 	"os"
-
 	"strings"
 
 	"github.com/dutchcoders/transfer.sh/server"
 	"github.com/fatih/color"
 	"github.com/minio/cli"
-	"log"
 	"google.golang.org/api/googleapi"
 )
 
@@ -117,6 +115,10 @@ var globalFlags = []cli.Flag{
 		Value:  "",
 		EnvVar: "BUCKET",
 	},
+	cli.BoolFlag{
+		Name:  "s3-no-multipart",
+		Usage: "Disables S3 Multipart Puts",
+	},
 	cli.StringFlag{
 		Name:  "gdrive-client-json-filepath",
 		Usage: "",
@@ -294,7 +296,7 @@ func New() *Cmd {
 				panic("secret-key not set.")
 			} else if bucket := c.String("bucket"); bucket == "" {
 				panic("bucket not set.")
-			} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger); err != nil {
+			} else if storage, err := server.NewS3Storage(accessKey, secretKey, bucket, c.String("s3-endpoint"), logger, c.Bool("s3-no-multipart")); err != nil {
 				panic(err)
 			} else {
 				options = append(options, server.UseStorage(storage))

server/storage.go

@@ -2,26 +2,25 @@ package server
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"mime"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 
-	"encoding/json"
 	"github.com/goamz/goamz/s3"
-
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
-	"io/ioutil"
-	"net/http"
-	"strings"
 )
 
 type Storage interface {
@@ -125,17 +124,18 @@ func (s *LocalStorage) Put(token string, filename string, reader io.Reader, cont
 
 type S3Storage struct {
 	Storage
-	bucket *s3.Bucket
+	bucket      *s3.Bucket
-	logger *log.Logger
+	logger      *log.Logger
+	noMultipart bool
 }
 
-func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger) (*S3Storage, error) {
+func NewS3Storage(accessKey, secretKey, bucketName, endpoint string, logger *log.Logger, disableMultipart bool) (*S3Storage, error) {
 	bucket, err := getBucket(accessKey, secretKey, bucketName, endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	return &S3Storage{bucket: bucket, logger: logger}, nil
+	return &S3Storage{bucket: bucket, logger: logger, noMultipart: disableMultipart}, nil
 }
 
 func (s *S3Storage) Type() string {
@@ -150,7 +150,6 @@ func (s *S3Storage) Head(token string, filename string) (contentType string, con
 	if err != nil {
 		return
 	}
-
 	contentType = response.Header.Get("Content-Type")
 
 	contentLength, err = strconv.ParseUint(response.Header.Get("Content-Length"), 10, 0)
@@ -202,8 +201,7 @@ func (s *S3Storage) Delete(token string, filename string) (err error) {
 	return
 }
 
-func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
+func (s *S3Storage) putMulti(key string, reader io.Reader, contentType string, contentLength uint64) (err error) {
-	key := fmt.Sprintf("%s/%s", token, filename)
 
 	var (
 		multi *s3.Multi
@@ -316,6 +314,25 @@ func (s *S3Storage) Put(token string, filename string, reader io.Reader, content
 	return
 }
 
+func (s *S3Storage) Put(token string, filename string, reader io.Reader, contentType string, contentLength uint64) (err error) {
+	key := fmt.Sprintf("%s/%s", token, filename)
+
+	s.logger.Printf("Uploading file %s to S3 Bucket", filename)
+	if !s.noMultipart {
+		err = s.putMulti(key, reader, contentType, contentLength)
+	} else {
+		err = s.bucket.PutReader(key, reader, int64(contentLength), contentType, s3.Private, s3.Options{})
+	}
+
+	if err != nil {
+		return
+	}
+
+	s.logger.Printf("Completed uploading %s", filename)
+
+	return
+}
+
 type GDrive struct {
 	service         *drive.Service
 	rootId          string