From 9b937ae6b3ba75936dda6a34b5f7e92406f91ec3 Mon Sep 17 00:00:00 2001 From: coffeegoddd Date: Mon, 29 Apr 2024 10:40:21 -0700 Subject: [PATCH] /{README.md,go,scripts}: update readme, add scripts and smtp_connection_helper source --- README.md | 19 +- go/cmd/smtp_connection_helper/main.go | 276 ++++++++++++++++++++++++++ go/go.mod | 8 + go/go.sum | 9 + scripts/centos_install.sh | 87 ++++++++ scripts/ubuntu_install.sh | 90 +++++++++ 6 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 go/cmd/smtp_connection_helper/main.go create mode 100644 go/go.mod create mode 100644 go/go.sum create mode 100644 scripts/centos_install.sh create mode 100644 scripts/ubuntu_install.sh diff --git a/README.md b/README.md index d16883c..c777561 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,19 @@ # doltlab-issues -Issue tracking for DoltLab + +DoltLab is currently closed source. +This repo is used for tracking issues, publishing release notes, and tracking some auxillary DoltLab tools and scripts. + +For more general DoltLab information, check out [DoltLab's documentation site](https://docs.doltlab.com) or DoltHub's [blog](https://www.dolthub.com/blog). + +# Installer scripts + +We've written some scripts to make installing DoltLab's dependencies easier. + +- [Install dependencies on Ubuntu](./scripts/ubuntu_install.sh) +- [Install dependencies on CentOS](./scripts/centos_install.sh) + +# Tools + +Included in DoltLab's releases are some helpful tools written in `go`. You can find the source for these tools in the `go/cmd` package. + +- [smtp_connection_helper](./go/cmd/smtp_connection_helper/main.go). diff --git a/go/cmd/smtp_connection_helper/main.go b/go/cmd/smtp_connection_helper/main.go new file mode 100644 index 0000000..61cce69 --- /dev/null +++ b/go/cmd/smtp_connection_helper/main.go @@ -0,0 +1,276 @@ +package main + +import ( + "crypto/tls" + "errors" + "flag" + "fmt" + "io" + "log" + "strings" + + "github.com/emersion/go-sasl" + "github.com/emersion/go-smtp" +) + +const ( + AuthPlain = "plain" + AuthAnonymous = "anonymous" + AuthExternal = "external" + AuthOauthBearer = "oauthbearer" + AuthDisable = "disable" + AuthLogin = "login" +) + +var sender = flag.String("from", "", "email address of sender") +var recipient = flag.String("to", "", "email address of recipient") +var host = flag.String("host", "", "smtp host") +var port = flag.Int("port", 0, "smtp port") +var auth = flag.String("auth", "", fmt.Sprintf("authentication method, one of '%s', '%s', '%s', '%s', '%s', '%s'", AuthPlain, AuthLogin, AuthAnonymous, AuthExternal, AuthOauthBearer, AuthDisable)) +var implicitTLSArg = flag.Bool("implicit-tls", false, "use TLS Wrapper connection instead of STARTTLS connection") +var insecure = flag.Bool("insecure", false, "if used, sets TLS Config InsecureSkipVerify to true") +var username = flag.String("username", "", "smtp username") +var password = flag.String("password", "", "smtp password") +var identity = flag.String("identity", "", "smtp identity") +var token = flag.String("token", "", "oauthbearer token") +var trace = flag.String("trace", "", "trace argument passed to anonymous smtp client") +var subject = flag.String("subject", "Testing SMTP Server Connection", "email subject") +var message = flag.String("message", "This is a test email message sent with smtp_connection_helper!", "message to send to recipient") +var clientHostnameArg = flag.String("client-hostname", "localhost", "specifies client hostname passed SMTP server during HELO or EHLO") +var help = flag.Bool("help", false, "print 'smtp_connection_helper' usage") + +func main() { + flag.Parse() + + if *help { + printUsage() + } else { + + if *host == "" || *port < 1 || *sender == "" || *recipient == "" || *subject == "" || *message == "" || *clientHostnameArg == "" { + printUsage() + log.Fatal(errors.New("one or all required arguments not supplied, must supply: --host, --port, --from, --to, --subject, --message, --client-hostname")) + } + + var err error + switch { + case *auth == AuthPlain: + err = sendEmailPlainAuth(*host, *port, *identity, *username, *password, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + case *auth == AuthLogin: + err = sendEmailLoginAuth(*host, *port, *username, *password, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + case *auth == AuthAnonymous: + err = sendEmailAnonymousAuth(*host, *port, *trace, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + case *auth == AuthExternal: + err = sendEmailExternalAuth(*host, *port, *identity, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + case *auth == AuthOauthBearer: + err = sendEmailOauthBearerAuth(*host, *port, *username, *token, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + case *auth == AuthDisable: + err = sendEmailDisableAuth(*host, *port, *sender, *recipient, *subject, *message, *clientHostnameArg, *implicitTLSArg, *insecure) + default: + err = errors.New(fmt.Sprintf("Unsupported auth method: %s", *auth)) + } + + if err != nil { + log.Fatal(err) + } + + fmt.Println("Successfully sent email!") + } +} + +func sendEmailPlainAuth(host string, port int, identity, username, password, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + if username == "" || password == "" { + return errors.New(fmt.Sprintf("username and password must not be empty for auth %s", AuthPlain)) + } + + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + + authClient := sasl.NewPlainClient(identity, username, password) + return sendEmail(smtpClient, authClient, sender, recipient, subject, message, clientHostname, AuthPlain) +} + +func sendEmailLoginAuth(host string, port int, username, password, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + if username == "" || password == "" { + return errors.New(fmt.Sprintf("username and password must not be empty for auth %s", AuthLogin)) + } + + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + + authClient := sasl.NewLoginClient(username, password) + return sendEmail(smtpClient, authClient, sender, recipient, subject, message, clientHostname, AuthLogin) +} + +func sendEmailAnonymousAuth(host string, port int, trace, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + + authClient := sasl.NewAnonymousClient(trace) + return sendEmail(smtpClient, authClient, sender, recipient, subject, message, clientHostname, AuthAnonymous) +} + +func sendEmailExternalAuth(host string, port int, identity, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + + authClient := sasl.NewExternalClient(identity) + return sendEmail(smtpClient, authClient, sender, recipient, subject, message, clientHostname, AuthExternal) +} + +func sendEmailOauthBearerAuth(host string, port int, username, token, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + if username == "" || token == "" { + return errors.New(fmt.Sprintf("username and token must not be empty for auth %s", AuthOauthBearer)) + } + + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + + authClient := sasl.NewOAuthBearerClient(&sasl.OAuthBearerOptions{ + Username: username, + Token: token, + Host: host, + Port: port, + }) + + return sendEmail(smtpClient, authClient, sender, recipient, subject, message, clientHostname, AuthOauthBearer) +} + +func sendEmailDisableAuth(host string, port int, sender, recipient, subject, message, clientHostname string, implicitTLS, insecure bool) error { + smtpClient, err := getSmtpClient(host, port, implicitTLS, insecure) + if err != nil { + return err + } + return sendEmail(smtpClient, nil, sender, recipient, subject, message, clientHostname, AuthDisable) +} + +func getSmtpClient(host string, port int, implicitTLS, insecure bool) (*smtp.Client, error) { + if implicitTLS { + return newImplicitTLSClient(host, port, insecure) + } + return newExplicitStartTLSClient(host, port) +} + +func newImplicitTLSClient(host string, port int, insecure bool) (*smtp.Client, error) { + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%d", host, port), &tls.Config{ + ServerName: host, + InsecureSkipVerify: insecure, + }) + if err != nil { + return nil, err + } + return smtp.NewClient(conn), nil +} + +func newExplicitStartTLSClient(host string, port int) (*smtp.Client, error) { + return smtp.Dial(fmt.Sprintf("%s:%d", host, port)) +} + +func sendEmail(smtpClient *smtp.Client, authClient sasl.Client, sender, recipient, subject, message, clientHostname string, method string) (err error) { + defer func() { + rerr := smtpClient.Close() + if err == nil { + if rerr.Error() != "use of closed network connection" { + err = rerr + } + + } + }() + + body := "MIME-version: 1.0;\nContent-Type: text/html; charset=\"UTF-8\";\r\n" + body += fmt.Sprintf("From: %s\r\n", sender) + body += fmt.Sprintf("To: %s\r\n", recipient) + body += fmt.Sprintf("Subject: %s\r\n", subject) + body += fmt.Sprintf("\r\n%s\r\n", message) + + fmt.Println("Sending email with auth method:", method) + + err = smtpClient.Hello(clientHostname) + if err != nil { + return + } + + if ok, _ := smtpClient.Extension("STARTTLS"); ok { + if err = smtpClient.StartTLS(nil); err != nil { + return + } + } + + if authClient != nil { + ok, _ := smtpClient.Extension("AUTH") + if !ok { + return errors.New("smtp: server doesn't support AUTH") + } else { + err = smtpClient.Auth(authClient) + if err != nil { + return + } + } + } + + err = smtpClient.Mail(sender, nil) + if err != nil { + return + } + + err = smtpClient.Rcpt(recipient, nil) + if err != nil { + return + } + + var wc io.WriteCloser + wc, err = smtpClient.Data() + if err != nil { + return + } + + _, err = io.Copy(wc, strings.NewReader(body)) + if err != nil { + return + } + + err = wc.Close() + if err != nil { + return + } + + return smtpClient.Quit() +} + +func printUsage() { + fmt.Println("") + fmt.Println("") + fmt.Println("'smtp_connection_helper' is a simple tool used to ensure you can successfully connect to an smtp server.") + fmt.Println("If the connection is successful, this tool will send a test email to a single recipient from a single sender.") + fmt.Println("By default 'smtp_connection_helper' will attempt to connect to the SMTP server with STARTTLS. To use implicit TLS, use --implicit-tls") + fmt.Println("") + fmt.Println(fmt.Sprintf("Usage:")) + fmt.Println("") + fmt.Println(fmt.Sprintf("./smtp_connection_helper \\")) + fmt.Println("--host \\") + fmt.Println("--port \\") + fmt.Println("--from \\") + fmt.Println("--to \\") + fmt.Println("--message {This is a test email message sent with smtp_connection_helper!} \\") + fmt.Println("--subject {Testing SMTP Server Connection} \\") + fmt.Println("--client-hostname {localhost} \\") + fmt.Println(fmt.Sprintf("--auth <%s|%s|%s|%s|%s|%s> \\", AuthPlain, AuthLogin, AuthExternal, AuthAnonymous, AuthOauthBearer, AuthDisable)) + fmt.Println("[--username smtp username] \\") + fmt.Println("[--password smtp password] \\") + fmt.Println("[--token smtp oauth token] \\") + fmt.Println("[--identity smtp identity] \\") + fmt.Println("[--trace anonymous trace] \\") + fmt.Println("[--implicit-tls] \\") + fmt.Println("[--insecure]") + fmt.Println("") + fmt.Println("") +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..557a6ec --- /dev/null +++ b/go/go.mod @@ -0,0 +1,8 @@ +module github.com/dolthub/doltlab-issues + +go 1.22.1 + +require ( + github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac + github.com/emersion/go-smtp v0.20.2 +) diff --git a/go/go.sum b/go/go.sum new file mode 100644 index 0000000..1af747b --- /dev/null +++ b/go/go.sum @@ -0,0 +1,9 @@ +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac h1:tn/OQ2PmwQ0XFVgAHfjlLyqMewry25Rz7jWnVoh4Ggs= +github.com/emersion/go-sasl v0.0.0-20211008083017-0b9dcfb154ac/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43 h1:hH4PQfOndHDlpzYfLAAfl63E8Le6F2+EL/cdhlkyRJY= +github.com/emersion/go-sasl v0.0.0-20231106173351-e73c9f7bad43/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-smtp v0.20.2 h1:peX42Qnh5Q0q3vrAnRy43R/JwTnnv75AebxbkTL7Ia4= +github.com/emersion/go-smtp v0.20.2/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= +github.com/emersion/go-smtp v0.21.1 h1:VQeZSZAKk8ueYii1yR5Zalmy7jI287eWDUqSaJ68vRM= +github.com/emersion/go-smtp v0.21.1/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= diff --git a/scripts/centos_install.sh b/scripts/centos_install.sh new file mode 100644 index 0000000..e8763e9 --- /dev/null +++ b/scripts/centos_install.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -e +set -o pipefail + +version="" + +if [ "$#" -eq 1 ]; then + echo "" + echo "Preparing to install DoltLab $1" + echo "" + version="$1" +else + echo "" + echo "Preparing to install the latest DoltLab" + echo "" + version="latest" +fi + +with_sudo="sudo" +user="$(whoami)" + +export USER=${USER-$user} +export PATH=$PATH:/usr/local/bin + +export DEBIAN_FRONTEND=noninteractive + +eval sudo -V > /dev/null 2>&1 || with_sudo="" && true + +# download doltlab +curl -LO https://doltlab-releases.s3.amazonaws.com/linux/amd64/doltlab-"$version".zip + +# create docker group if it doesnt exist +group=docker +eval "$with_sudo getent group $group" || eval "$with_sudo groupadd $group" + +# do this here to avoid 'newgrp' command +# which doesnt work well in scripts +if [ $(id -gn) != $group ]; then + eval exec "$with_sudo" sg $group "$0 $*" +fi + +echo "Preparing to download DoltLab $version" + +# install tools make, unzip, git +eval "$with_sudo yum makecache -y" +eval "$with_sudo yum install unzip make git -y" + +# install docker and docker-compose +eval "$with_sudo yum install -y yum-utils" +eval "$with_sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo" +eval "$with_sudo yum install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin" +eval "$with_sudo systemctl start docker" + +# sanity check +docker --version + +# sanity check +compose_command="docker compose" +eval "$compose_command" version > /dev/null 2>&1 || compose_command="docker-compose" && true +eval "$compose_command version" + +eval "$with_sudo usermod -aG docker $USER" + +# sanity check +docker ps + +# install creds-helper and create config +git clone https://github.com/awslabs/amazon-ecr-credential-helper.git +cd amazon-ecr-credential-helper && make docker +eval "$with_sudo mv ./bin/local/docker-credential-ecr-login /usr/local/bin/" + +docker-credential-ecr-login -v +cd .. && mkdir -p ~/.docker +echo '{"credHelpers":{"public.ecr.aws":"ecr-login"}}' > ~/.docker/config.json + +# unzip DoltLab +unzip doltlab-"$version".zip -d doltlab + +echo "" +echo "" +echo "All dependencies installed successfully" +echo "" +echo "DoltLab $version has been download and unzipped to: ./doltlab" +echo "" +echo "Please run 'sudo newgrp docker' to use docker without 'sudo'" +echo "" diff --git a/scripts/ubuntu_install.sh b/scripts/ubuntu_install.sh new file mode 100644 index 0000000..2abba32 --- /dev/null +++ b/scripts/ubuntu_install.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +set -e +set -o pipefail + +version="" + +if [ "$#" -eq 1 ]; then + echo "" + echo "Preparing to install DoltLab $1" + echo "" + version="$1" +else + echo "" + echo "Preparing to install the latest DoltLab" + echo "" + version="latest" +fi + +with_sudo="sudo" +user="$(whoami)" + +eval sudo -V > /dev/null 2>&1 || with_sudo="" && true + +export DEBIAN_FRONTEND=noninteractive +export USER=${USER-$user} + +# download doltlab +curl -LO https://doltlab-releases.s3.amazonaws.com/linux/amd64/doltlab-"$version".zip + +# create docker group if it doesnt exist +group=docker +eval "$with_sudo getent group $group" || eval "$with_sudo groupadd $group" + +# do this here to avoid 'newgrp' command +# which doesnt work well in scripts +if [ $(id -gn) != $group ]; then + eval exec "$with_sudo" sg $group "$0 $*" +fi + +echo "Preparing to download DoltLab $version" + +# install tools make and unzip +eval "$with_sudo apt update -y" +eval "$with_sudo apt install -y make unzip" + +# install docker and docker-compose +eval "$with_sudo apt-get update -y" +eval "$with_sudo apt-get install -y ca-certificates curl gnupg lsb-release" + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | eval "$with_sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg" +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | eval "$with_sudo tee /etc/apt/sources.list.d/docker.list" > /dev/null +eval "$with_sudo apt-get update -y" +eval "$with_sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin" + +# sanity check +docker --version + +# sanity check +compose_command="docker compose" +eval "$compose_command" version > /dev/null 2>&1 || compose_command="docker-compose" && true +eval "$compose_command version" + +eval "$with_sudo getent group $group" || eval "$with_sudo groupadd $group" +eval "$with_sudo usermod -aG docker $USER" + +# sanity check +docker ps + +# install creds-helper and create config +git clone https://github.com/awslabs/amazon-ecr-credential-helper.git +cd amazon-ecr-credential-helper && make docker +eval "$with_sudo mv ./bin/local/docker-credential-ecr-login /usr/local/bin/" +docker-credential-ecr-login -v +cd .. && mkdir -p ~/.docker +echo '{"credHelpers":{"public.ecr.aws":"ecr-login"}}' > ~/.docker/config.json + +# unzip DoltLab +unzip doltlab-"$version".zip -d doltlab + +echo "" +echo "" +echo "All dependencies installed successfully" +echo "" +echo "DoltLab $version has been download and unzipped to: ./doltlab" +echo "" +echo "Please run 'sudo newgrp docker' to use docker without 'sudo'" +echo ""