In this example we are going to use a SFTP client and server to upload/download files over a SSH connection. For the server simulation we will create a Docker container which provides basic functionality to interact with the server.


Important: Prefer this version over the previous version where file is read into memory as it is not memory efficient.


Structure


├── docker
│   └── docker-compose.yaml
├── main.go
├── file.txt
├── sftp
│   └── sftp.go
├── ssh
│   └── id_rsa
└── tmp

Files


docker-compose.yaml


The private key authentication requires copying SSH keys. Depending on your SSH setup, use $ cp ~/.ssh/[id_rsa|id_ed25519] ssh/ command before running the docker command. I have id_rsa file hence reason using it.


version: "3.4"
services:
sftp-server:
image: "atmoz/sftp"
container_name: "sftp-server"
ports:
- "2022:22"
volumes:
- "../tmp:/home/inanzzz/tmp"
- "$HOME/.ssh/id_rsa:/etc/ssh/ssh_host_rsa_key:ro"
- "$HOME/.ssh/id_rsa.pub:/home/inanzzz/.ssh/keys/id_rsa.pub:ro"
# - "$HOME/.ssh/id_ed25519:/etc/ssh/ssh_host_ed25519_key:ro"
# - "$HOME/.ssh/id_ed25519.pub:/home/inanzzz/.ssh/keys/id_ed25519.pub:ro"
command: "inanzzz:password:1001"

sftp.go


Behind the scene it establishes a new SSH connection with a remote server that powers the SFTP client. Due to lack of built-in "keepalive" and auto reconnect options, every call to Upload, Download and Info functions open up a new SSH connection only if it was closed otherwise reuses live one. It currently leverages "password" and "private key" authentication methods which are configured with the Config type. If both enabled at same time, the private key authentication takes precedence.


package sftp

import (
"fmt"
"io"
"net"
"os"
"time"

"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)

// Config represents SSH connection parameters.
type Config struct {
Username string
Password string
PrivateKey string
Server string
KeyExchanges []string

Timeout time.Duration
}

// Client provides basic functionality to interact with a SFTP server.
type Client struct {
config Config
sshClient *ssh.Client
sftpClient *sftp.Client
}

// New initialises SSH and SFTP clients and returns Client type to use.
func New(config Config) (*Client, error) {
c := &Client{
config: config,
}

if err := c.connect(); err != nil {
return nil, err
}

return c, nil
}

// Create creates a remote/destination file for I/O.
func (c *Client) Create(filePath string) (io.ReadWriteCloser, error) {
if err := c.connect(); err != nil {
return nil, fmt.Errorf("connect: %w", err)
}

return c.sftpClient.Create(filePath)
}

// Upload writes local/source file data streams to remote/destination file.
func (c *Client) Upload(source io.Reader, destination io.Writer, size int) error {
if err := c.connect(); err != nil {
return fmt.Errorf("connect: %w", err)
}

chunk := make([]byte, size)

for {
num, err := source.Read(chunk)
if err == io.EOF {
tot, err := destination.Write(chunk[:num])
if err != nil {
return err
}

if tot != len(chunk[:num]) {
return fmt.Errorf("failed to write stream")
}

return nil
}

if err != nil {
return err
}

tot, err := destination.Write(chunk[:num])
if err != nil {
return err
}

if tot != len(chunk[:num]) {
return fmt.Errorf("failed to write stream")
}
}
}

// Download returns remote/destination file for reading.
func (c *Client) Download(filePath string) (io.ReadCloser, error) {
if err := c.connect(); err != nil {
return nil, fmt.Errorf("connect: %w", err)
}

return c.sftpClient.Open(filePath)
}

// Info gets the details of a file. If the file was not found, an error is returned.
func (c *Client) Info(filePath string) (os.FileInfo, error) {
if err := c.connect(); err != nil {
return nil, fmt.Errorf("connect: %w", err)
}

info, err := c.sftpClient.Lstat(filePath)
if err != nil {
return nil, fmt.Errorf("file stats: %w", err)
}

return info, nil
}

// Close closes open connections.
func (c *Client) Close() {
if c.sftpClient != nil {
c.sftpClient.Close()
}
if c.sshClient != nil {
c.sshClient.Close()
}
}

// connect initialises a new SSH and SFTP client only if they were not
// initialised before at all and, they were initialised but the SSH
// connection was lost for any reason.
func (c *Client) connect() error {
if c.sshClient != nil {
_, _, err := c.sshClient.SendRequest("keepalive", false, nil)
if err == nil {
return nil
}
}

auth := ssh.Password(c.config.Password)
if c.config.PrivateKey != "" {
signer, err := ssh.ParsePrivateKey([]byte(c.config.PrivateKey))
if err != nil {
return fmt.Errorf("ssh parse private key: %w", err)
}
auth = ssh.PublicKeys(signer)
}

cfg := &ssh.ClientConfig{
User: c.config.Username,
Auth: []ssh.AuthMethod{
auth,
},
HostKeyCallback: func(string, net.Addr, ssh.PublicKey) error { return nil },
Timeout: c.config.Timeout,
Config: ssh.Config{
KeyExchanges: c.config.KeyExchanges,
},
}

sshClient, err := ssh.Dial("tcp", c.config.Server, cfg)
if err != nil {
return fmt.Errorf("ssh dial: %w", err)
}
c.sshClient = sshClient

sftpClient, err := sftp.NewClient(sshClient)
if err != nil {
return fmt.Errorf("sftp new client: %w", err)
}
c.sftpClient = sftpClient

return nil
}

main.go


package main

import (
"fmt"
"io/ioutil"
"log"
"os"
"time"

"github.com/you/client/sftp"
)

func main() {
pk, err := ioutil.ReadFile("./ssh/id_rsa") // required only if private key authentication is to be used
if err != nil {
log.Fatalln(err)
}

config := sftp.Config{
Username: "inanzzz",
Password: "password", // required only if password authentication is to be used
PrivateKey: string(pk), // required only if private key authentication is to be used
Server: "0.0.0.0:2022",
KeyExchanges: []string{"diffie-hellman-group-exchange-sha256", "diffie-hellman-group14-sha256"}, // optional
Timeout: time.Second * 30, // 0 for not timeout
}

client, err := sftp.New(config)
if err != nil {
log.Fatalln(err)
}
defer client.Close()

// Open local file for reading.
source, err := os.Open("file.txt")
if err != nil {
log.Fatalln(err)
}
defer source.Close()

// Create remote file for writing.
destination, err := client.Create("tmp/file.txt")
if err != nil {
log.Fatalln(err)
}
defer destination.Close()

// Upload local file to a remote location as in 1MB (byte) chunks.
if err := client.Upload(source, destination, 1000000); err != nil {
log.Fatalln(err)
}

// Download remote file.
file, err := client.Download("tmp/file.txt")
if err != nil {
log.Fatalln(err)
}
defer file.Close()

// Read downloaded file.
data, err := ioutil.ReadAll(file)
if err != nil {
log.Fatalln(err)
}
fmt.Println(string(data))

// Get remote file stats.
info, err := client.Info("tmp/file.txt")
if err != nil {
log.Fatalln(err)
}
fmt.Printf("%+v\n", info)
}

Test


Docker


$ docker-compose up

Recreating sftp-server ... done
Attaching to sftp-server
sftp-server | [/usr/local/bin/create-sftp-user] Parsing user data: "inanzzz:password:1001"
sftp-server | Generating public/private ed25519 key pair.
sftp-server | Your identification has been saved in /etc/ssh/ssh_host_ed25519_key.
sftp-server | Your public key has been saved in /etc/ssh/ssh_host_ed25519_key.pub.
sftp-server | The key fingerprint is:
sftp-server | SHA256:ggGrRmCvYias/aso5Lsad3POMerwIzys9fseI+4Zweghh root@111f66e033b4
sftp-server | The keys randomart image is:
sftp-server | +--[ED25519 256]--+
sftp-server | | o |
sftp-server | | . o |
sftp-server | |o o o . |
sftp-server | |=++o+o o + |
sftp-server | |+=*=+.. + |
sftp-server | |O*=o..o . |
sftp-server | +----[SHA256]-----+
sftp-server | chmod: changing permissions of '/etc/ssh/ssh_host_rsa_key': Read-only file system
sftp-server | [/entrypoint] Executing sshd
sftp-server | Server listening on 0.0.0.0 port 22.
sftp-server | Server listening on :: port 22.

Run


$ go run -race main.go
File Content
&{name:file.txt stat:0xc0001d01c0}

// When using private key auth
sftp-server | Accepted publickey for inanzzz from 172.21.0.1 port 57818 ssh2: RSA SHA256:saboQipddaHVp67cUsadTSwS1ORslJioasasCA6NIDs
// When using password auth
sftp-server | Accepted password for inanzzz from 172.21.0.1 port 57822 ssh2