diff --git a/addresses.go b/addresses.go index 16682eb..66898cc 100644 --- a/addresses.go +++ b/addresses.go @@ -1,89 +1,92 @@ /* - Copyright © 2018 Harald Sitter + Copyright © 2018-2019 Harald Sitter This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License or any later version accepted by the membership of KDE e.V. (or its successor approved by the membership of KDE e.V.), which shall act as a proxy defined in Section 14 of version 3 of the license. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ package main import ( "fmt" "sync" "time" ) // Could/should use atomic really. the manual mutex playing is meh var _addresses []string var _addressesMutex sync.RWMutex +// GetAddresses returns a slice of strings which are known and trusted +// addresses. Remotes which do not match the addresses in the slice are +// considered foreigh and shouldn't be talked to. func GetAddresses() []string { // NB: we don't need to manually copy this or anything. The slice is fixed // once it is assigned to _addresses, we only need the mutex to ensure the // var doesn't get assigned while we read it. Once we have the slice // we are good to go. _addressesMutex.RLock() ret := _addresses _addressesMutex.RUnlock() return ret } func addressReset() { _addressesMutex.Lock() _addresses = []string{} _addressesMutex.Unlock() } func addressPoll() { addresses := []string{} // If there was an error, enter startup state. Errors mustn't be fatal // to prevent breakage in production. // This is particularly important in case the API endpoints go down for // whatever reason. list, err := dropletsPoll() if err != nil { fmt.Println(err) addressReset() return } addresses = append(addresses, list...) list, err = packetsPoll() addresses = append(addresses, list...) if err != nil { fmt.Println(err) addressReset() return } _addressesMutex.Lock() _addresses = addresses _addressesMutex.Unlock() } func addressesPollTick() { // This would cause 360 polls an hour, we have a limit of 5000 requests per // hour on the DO API side, so we should be well within the limit here. pollTicker := time.NewTicker(10 * time.Second) go func() { for { addressPoll() <-pollTicker.C } }() } diff --git a/deploy.sh b/deploy.sh index 86b1322..583b31c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,22 +1,27 @@ #!/bin/bash # script to deploy the sftp-bridge to the charlotte export GOPATH=$HOME echo -n "Making the data directory ... " mkdir -p $HOME/data chmod 700 $HOME/data echo "[done]" echo "Starting installation of the neon-sftp-bridge ... " go get -v -u anongit.kde.org/sysadmin/neon-sftp-bridge.git ln -fs $GOPATH/bin/neon-sftp-bridge.git $GOPATH/bin/neon-sftp-bridge echo "[done]" echo -n "Installing systemd service files ... " SYSTMEDUSERDIR=$HOME/.config/systemd/user mkdir -p $SYSTMEDUSERDIR -cp neon-sftp-bridge.service $SYSTMEDUSERDIR/ +cp systemd/neon-sftp-bridge.service $SYSTMEDUSERDIR/ +cp systemd/neon-sftp-bridge.socket $SYSTMEDUSERDIR/ +systemctl --user daemon-reload systemctl --user enable neon-sftp-bridge.service -systemctl --user start neon-sftp-bridge.service +systemctl --user enable neon-sftp-bridge.socket +systemctl --user reload neon-sftp-bridge.socket +systemctl --user stop neon-sftp-bridge.service echo "[done]" + diff --git a/main.go b/main.go index 96ade66..4710cc9 100644 --- a/main.go +++ b/main.go @@ -1,241 +1,250 @@ /* - Copyright 2016-2018 Harald Sitter + Copyright 2016-2019 Harald Sitter This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License or any later version accepted by the membership of KDE e.V. (or its successor approved by the membership of KDE e.V.), which shall act as a proxy defined in Section 14 of version 3 of the license. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ package main import ( "bufio" "bytes" + "context" "fmt" "io" "io/ioutil" "log" "net" "os" + "os/signal" "path" "path/filepath" "strings" + "syscall" + "time" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" + "github.com/coreos/go-systemd/activation" "github.com/gin-gonic/gin" "github.com/pkg/sftp" "net/http" _ "net/http/pprof" ) func isHidden(fName string) bool { return strings.HasPrefix(fName, ".") } func getFile(c *gin.Context, sftp *sftp.Client, path string) { fmt.Println("file") file, err := sftp.Open(path) defer file.Close() if err != nil { panic(err) } buffer := bufio.NewReader(file) // For unknown reasons reading from sftp files never EOFs, so // we need to manually keep track of how much we can and have read and abort // once all bytes are read. stat, err := file.Stat() if err != nil { panic(err) } toRead := stat.Size() c.Stream(func(w io.Writer) bool { wrote, err := buffer.WriteTo(w) toRead -= wrote if err != nil || toRead <= 0 { return false } return true }) } func getDir(c *gin.Context, sftp *sftp.Client, path string) { fmt.Println("dir") fileInfos, err := sftp.ReadDir(path) if err != nil { panic(err) } var buffer bytes.Buffer buffer.WriteString("") for _, info := range fileInfos { url := info.Name() if isHidden(url) { continue } buffer.WriteString(fmt.Sprintf("%s
\n", url, url)) } buffer.WriteString("") c.Data(http.StatusOK, "text/html", buffer.Bytes()) } func newSession() (*ssh.Client, *sftp.Client) { - // key, err := ioutil.ReadFile(filepath.Join(os.Getenv("HOME"), ".ssh/keys/kde.depot-8192")) + // key, err := ioutil.ReadFile(filepath.Join(os.Getenv("HOME"), ".ssh/keys/kde-8192")) key, err := ioutil.ReadFile(filepath.Join(os.Getenv("HOME"), ".ssh/id_rsa")) if err != nil { log.Fatalf("unable to read private key: %v", err) } // Create the Signer for this private key. signer, err := ssh.ParsePrivateKey(key) if err != nil { log.Fatalf("unable to parse private key: %v", err) } hostKeyCallback, err := knownhosts.New(filepath.Join(os.Getenv("HOME"), ".ssh/known_hosts")) if err != nil { log.Fatalf("failed to create known_hosts handler: %v", err) } config := &ssh.ClientConfig{ User: "ftpneon", Auth: []ssh.AuthMethod{ ssh.PublicKeys(signer), }, HostKeyCallback: hostKeyCallback, } client, err := ssh.Dial("tcp", "master.kde.org:22", config) if err != nil { log.Fatalf("unable to connect: %v", err) } sftp, err := sftp.NewClient(client) if err != nil { log.Fatal(err) } return client, sftp } func knownIP(c *gin.Context) bool { ip := c.ClientIP() knownAddresses := GetAddresses() if len(knownAddresses) <= 0 { // Allow by default to not impair production services should the IP // listing break at some point. return true } res := false for _, addr := range knownAddresses { if !net.ParseIP(ip).Equal(net.ParseIP(addr)) { continue } res = true break } if !res { // Log the checked IP and all server IPs to allow investigating after the // fact if this should go wrong at some point. fmt.Printf("Stranger danger: %s not in %s\n", ip, knownAddresses) } return res } func allowed(c *gin.Context) bool { path := path.Clean(c.Param("path")) fmt.Println(path) return !strings.HasPrefix(path, "/.") && (strings.HasPrefix(path, "/stable") || strings.HasPrefix(path, "/unstable")) } func get(c *gin.Context) { if path.Clean(c.Param("path")) == "/robots.txt" { c.String(http.StatusOK, "User-agent: *\nDisallow: /") return } if !knownIP(c) { c.String(http.StatusForbidden, "STRANGER DANGER! Only trusted other servers are allowed!") return } if !allowed(c) { c.String(http.StatusForbidden, "not an allowed path") return } path := "/home/ftpneon/" + c.Param("path") ssh, sftp := newSession() defer ssh.Close() defer sftp.Close() fileInfo, err := sftp.Stat(path) if err != nil { c.String(http.StatusNotFound, err.Error()) } if fileInfo.IsDir() { getDir(c, sftp, path) } else { getFile(c, sftp, path) } } func main() { addressesPollTick() router := gin.Default() router.GET("*path", get) - port := os.Getenv("PORT") - if len(port) <= 0 { - port = "8080" + listeners, err := activation.Listeners() + if err != nil { + panic(err) } - iface := os.Getenv("INTERFACE") - if len(iface) > 0 { - go func() { - router.Run(iface + ":" + port) - }() + log.Println("starting servers") + var servers []*http.Server + for _, listener := range listeners { + server := &http.Server{Handler: router} + go server.Serve(listener) + servers = append(servers, server) } - if iface != "0.0.0.0" { - // Only when not already listening on everything. Otherwise we'd have - // a bind-race, so either we claim 0.0.0.0 and listen everywhere or only - // on the selected interfaces below making it impossible to claim 0.0.0.0. + if len(servers) == 0 { + panic("no systemd listeners set") + } - go func() { - // Docker interface - router.Run("172.17.0.1:" + port) - }() + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - go func() { - // Debian localhost compat - // https://www.debian.org/doc/manuals/debian-reference/ch05.en.html#_the_hostname_resolution - // https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=719621 - router.Run("127.0.1.1:" + port) - }() + // Wait for some quit cause. + // This could be INT, TERM, QUIT + // We'll then do a zero downtime shutdown. + // This relies on systemd managing the socket and us doing graceful listener + // shutdown. Once we are no longer listening, the system starts backlogging + // the socket until we get restarted and listen again. + // Ideally this results in zero dropped connections. + <-quit + log.Println("servers are shutting down") - go func() { - // Localhost - router.Run("127.0.0.1:" + port) - }() + for _, srv := range servers { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + srv.SetKeepAlivesEnabled(false) + if err := srv.Shutdown(ctx); err != nil { + log.Fatalf("Server Shutdown: %s", err) + } } - select {} // sleep thread forever + log.Println("Server exiting") } diff --git a/neon-sftp-bridge.service b/systemd/neon-sftp-bridge.service similarity index 57% rename from neon-sftp-bridge.service rename to systemd/neon-sftp-bridge.service index 60b1ed9..5852595 100644 --- a/neon-sftp-bridge.service +++ b/systemd/neon-sftp-bridge.service @@ -1,15 +1,12 @@ [Unit] Description=Neon SFTP Bridge to milonia.kde.org [Service] +Environment=GIN_MODE=release WorkingDirectory=/home/neonsftpproxy/data -Environment=PORT=9191 -# Listen everywhere. Scaled nodes are in a different DO team and thus -# don't have access to our private network. -Environment=INTERFACE=0.0.0.0 ExecStart=/home/neonsftpproxy/bin/neon-sftp-bridge Restart=always RestartSec=4 [Install] WantedBy=default.target diff --git a/systemd/neon-sftp-bridge.socket b/systemd/neon-sftp-bridge.socket new file mode 100644 index 0000000..5d2a5ee --- /dev/null +++ b/systemd/neon-sftp-bridge.socket @@ -0,0 +1,10 @@ +[Unit] +Description=download.kde.internal.neon.kde.org sftp to http bridge + +[Socket] +# Listen everywhere the bridge queries the various cloud APIs for servers +# which are allowed to talk to it. +ListenStream=0.0.0.0:9191 + +[Install] +WantedBy=sockets.target