From 2f588946481fb94cd92ae8a61f0f60fb0fae24b1 Mon Sep 17 00:00:00 2001 From: GRMrGecko Date: Mon, 20 Jul 2020 19:47:10 -0500 Subject: [PATCH] first commit --- .gitignore | 4 + License.txt | 19 ++ README.md | 148 ++++++++ api.go | 486 +++++++++++++++++++++++++++ config.go | 100 ++++++ database.go | 60 ++++ go.mod | 16 + go.sum | 58 ++++ http.go | 61 ++++ mail.go | 252 ++++++++++++++ main.go | 87 +++++ smtp.go | 107 ++++++ syslog.go | 299 +++++++++++++++++ websocket.go | 187 +++++++++++ websocket_handler.go | 86 +++++ www/bower.json | 31 ++ www/index.css | 134 ++++++++ www/index.html | 109 ++++++ www/index.js | 779 +++++++++++++++++++++++++++++++++++++++++++ 19 files changed, 3023 insertions(+) create mode 100644 .gitignore create mode 100644 License.txt create mode 100644 README.md create mode 100644 api.go create mode 100644 config.go create mode 100644 database.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 http.go create mode 100644 mail.go create mode 100644 main.go create mode 100644 smtp.go create mode 100644 syslog.go create mode 100644 websocket.go create mode 100644 websocket_handler.go create mode 100644 www/bower.json create mode 100644 www/index.css create mode 100644 www/index.html create mode 100644 www/index.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f2755e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +MailArchive.db +mail-archive +www/bower_components +config.json diff --git a/License.txt b/License.txt new file mode 100644 index 0000000..b1a385d --- /dev/null +++ b/License.txt @@ -0,0 +1,19 @@ +Copyright (c) 2020 Mr. Gecko's Media (James Coleman). http://mrgeckosmedia.com/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c24bfb0 --- /dev/null +++ b/README.md @@ -0,0 +1,148 @@ +# Mail Archive + +Mail Archive is a tool designed to store email copied to it for a limited period of time. It comes with a syslog server designed for use with an email spam gateway, such as Proxmox Mail Gateway, to store logs associated with an email alongside the email itself. The syslog server is designed for use with Postfix, however it can easily be adjusted to work with other email server log messages. + +## Use with a spam gateway + +1. Configure a (sub)domain with proper mx records for Mail Archive. +2. Setup Mail Archive with a configuration that meets your need; review the `config.go` file for available configurations. +3. Setup your email server to BCC all email to the Mail Archive server. +4. Setup your syslog server to send copies of messages over to Mail Archive. + +### Example configuration for rsyslog +``` +# Mail Archive syslog server +*.* @192.168.2.12:514 +``` + +## Use as a debug mail server + +Mail Archive can be used as a debug mail server for testing software fairly easily. + +### Example config + +```json +{ + "http_port": 1080, + "smtp_port": 1025, + "syslog_udp": false, + "ui_disable_spam_reporting": true, + "ui_disable_logs": true +} +``` + +After saving the config, simply start Mail Archive and configure your software accordingly. + +### Use with Ruby on Rails + +Edit `environments/development.rb`: + +```ruby +config.action_mailer.delivery_method = :smtp +config.action_mailer.smtp_settings = { :address => '127.0.0.1', :port => 1025 } +config.action_mailer.raise_delivery_errors = false +``` + +### Use with PHP + +You will have to use PHPMailer for this, as PHP is deisnged to use sendmail to send email. You can configure postfix to copy email to Mail Archive, but that gets complicated. + +```php +$mail = new PHPMailer(); + +$mail->IsSMTP(); +$mail->CharSet = 'UTF-8'; + +$mail->Host = "127.0.0.1"; +$mail->Port = 1025; +$mail->SMTPDebug = true; + +$mail->isHTML(true); +$mail->Subject = 'Here is the subject'; +$mail->Body = 'This is the HTML message body in bold!'; +$mail->AltBody = 'This is the body in plain text for non-HTML mail clients'; + +$mail->send(); +``` + +### Use with Django + +Add the following configuration to your project's `settings.py` + +```python +if DEBUG: + EMAIL_HOST = '127.0.0.1' + EMAIL_HOST_USER = '' + EMAIL_HOST_PASSWORD = '' + EMAIL_PORT = 1025 + EMAIL_USE_TLS = False +``` + +## API + +The API is available at path `/api` and is fairly feature rich. + +### /ping +Test to see that the server responds correctly. + +### /config + +Retrieve the configuration for the web UI and current message count. + +### /message_log + +Pull a list of messages from the message log along with metadata. + +Supported parameter: +| Parameter | Description | +| :-------: | :----------: | +| q | Search query | +| p | Page number | + +### /message/{id}.log + +Returns the log associated with a message. + +### /message/{id}.eml + +Returns the original email source. + +### /message/{id}.txt + +Returns the email's text body. + +### /message/{id}.html + +Returns the email's html body. + +### /message/{id}/learn_ham + +Report a message as ham to your spam reporting API. + +### /message/{id}/learn_spam + +Report a message as spam to your spam reporting API. + +### /message/{id} + +Pull metadata on a specific message. + +## Building + +There are a few items that must be gathered first before Mail Archive will work. + +### Bower Components + +[Bower](https://bower.io/) is a browser package manager which is installable via [npm](https://www.npmjs.com/). Once bower is available on your computer, simply go into the `www` directory and run the following. + +``` +bower install +``` + +### Building Mail Archive + +Go into the main directory for Mail Archive and run the following to build. You will need [golang](https://golang.org/) to build. + +``` +go build +``` diff --git a/api.go b/api.go new file mode 100644 index 0000000..c14ed2f --- /dev/null +++ b/api.go @@ -0,0 +1,486 @@ +package main + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "mime/quotedprintable" + "net/http" + "strconv" + "strings" + "time" + + "github.com/DusanKasan/parsemail" + "github.com/gorilla/mux" +) + +// Commonly used strings. +const ( + APIOK = "ok" + APIERR = "error" + APINoEndpoint = "No endpoint found" + APIReadMessage = "Error reading message." + APINoMessage = "Messages was not found" +) + +// Main response structure. +type APIGeneralResp struct { + Status string `json:"status"` + Error string `json:"error"` +} + +// Response with configuration. +type APIConfigResp struct { + APIGeneralResp + CustomBrand string `json:"custom_brand"` + DisableSpamReporting bool `json:"disable_spam_reporting"` + DisableLogs bool `json:"disable_logs"` + MessageCount uint `json:"message_count"` +} + +// Response with message log entries. +type APIMessageLogResp struct { + APIGeneralResp + Messages []MessageLog `json:"messages"` +} + +// Response with message entry. +type APIMessageEntryResp struct { + APIGeneralResp + Messages MessageLog `json:"message"` +} + +// Response to spam report requests. +type APISpamReportResp struct { + APIGeneralResp + Requests []map[string]string `json:"requests"` +} + +// Typical API responses are done with JSON. To make it easier to respond, this function will marshal/send json to a response writer. +func (s *HTTPServer) JSONResponse(w http.ResponseWriter, resp interface{}) { + // Encode response as json. + js, err := json.Marshal(resp) + if err != nil { + // Error should not happen normally... + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + return + } + + // If no error, we can set content type header and send response. + w.Header().Set("Content-Type", "application/json") + w.Write(js) +} + +// There are quite a few request that send a general response on error. This function is to make it easy to build/send a general response. +func (s *HTTPServer) APISendGeneralResp(w http.ResponseWriter, status, error string) { + resp := APIGeneralResp{} + resp.Status = status + resp.Error = error + s.JSONResponse(w, resp) +} + +// Setup HTTP router with routes for the API calls. +func (s *HTTPServer) RegisterAPIRoutes(r *mux.Router) { + api := r.PathPrefix("/api").Subrouter() + // Just a test call. + api.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) { + s.APISendGeneralResp(w, APIOK, "") + }) + + // Retrieve the configuration. + api.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) { + resp := APIConfigResp{} + resp.CustomBrand = app.config.UICustomBrand + resp.DisableSpamReporting = app.config.UIDisableSpamReporting + resp.DisableLogs = app.config.UIDisableLogs + resp.MessageCount = app.messageCount + s.JSONResponse(w, resp) + }) + + // Retrieve message logs matching criteria provided. + api.HandleFunc("/message_log", func(w http.ResponseWriter, r *http.Request) { + r.ParseForm() // r.Form isn't filled unless we first parse. + query := r.Form.Get("q") + + // Page variable provided should be an integer. + page, _ := strconv.Atoi(r.Form.Get("p")) + if page <= 0 { // If page is lower than 1, we need it to be page 1. + page = 1 + } + // Offset based on page number and max messages per page set. + offset := app.config.MessagesPerPage * (page - 1) + if offset <= 0 { // If lower than 1, we can just set to -1 to unset the offset field in queries. + offset = -1 + } + + var entries []MessageLog + // If a query is not provided, we can just pull entires from the database without additional filters. + if query == "" { + app.db.Order("received desc").Offset(offset).Limit(app.config.MessagesPerPage).Find(&entries) + } else { + // As a query was provided, we need to parse the query out to a SQL where statement. + // Splitting the query up by words to allow matches against 2 differen fields in the same query. + // Example: test@example.com sent + // The above will match both an email address and the status of sent. + queryS := strings.Split(query, " ") + var queries []string + var statements []interface{} // Must be an interface to expand to arguments in a function call. + // For each word, setup LIKE statements. + for _, q := range queryS { + likeStatement := "%" + q + "%" + // Append like queries to slice. + queries = append(queries, "(`from` LIKE ? OR `to` LIKE ? OR `subject` LIKE ? OR `source_ip` LIKE ? OR `message_id` LIKE ? OR `status` LIKE ?)") + // Append statements to slice. + statements = append(statements, likeStatement, likeStatement, likeStatement, likeStatement, likeStatement, likeStatement) + } + + // Join queries with an AND, and also turn statements into arguments for the database WHERE statement. + app.db.Where(strings.Join(queries, " AND "), statements...).Order("received desc").Offset(offset).Limit(app.config.MessagesPerPage).Find(&entries) + } + + // Return found entries, if any. + resp := APIMessageLogResp{} + resp.Status = APIOK + resp.Messages = entries + s.JSONResponse(w, resp) + }) + + // Pull log entries for a message. + api.HandleFunc("/message/{id}.log", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) // Parses the variable matched in the request URI. + UUID := vars["id"] + + // If message id provided is blank, the message cannot exist. + if UUID == "" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Search the database for a message log entry for the message id to ensure that it exists. + var messageEntry MessageLog + app.db.Where("uuid = ?", UUID).First(&messageEntry) + // If return UUID is blank, we didn't find an entry. + if messageEntry.UUID == "" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Search for SysLog ID information that matches the message ID. + var matches []SysLogIDInfo + app.db.Where("message_id = ?", messageEntry.MessageID).Find(&matches) + // If no matches, no logs. + if len(matches) == 0 { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Place all SIDs into a slice so that we can run a single query to get all log messages. + var sids []string + hostname := matches[0].Hostname // The hostname of the first entry should suffice here. We shouldn't see another host with that message id. + for _, match := range matches { + sids = append(sids, match.SID) + } + + // Find log messages orderd by timestamp matching the SIDs we gathered above. + var messages []SysLogMessage + app.db.Where("s_id IN (?) AND hostname = ?", sids, hostname).Order("timestamp").Find(&messages) + // If no log entries... We just can't provide them. + if len(messages) == 0 { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Log messages were found, we need content type to be plain text. + w.Header().Set("Content-Type", "text/plain") + // Print all logs to the response writer. + for _, message := range messages { + fmt.Fprintf(w, "%v %s %s: %s\n", message.Timestamp, message.Hostname, message.Tag, message.Content) + } + }) + + api.HandleFunc("/message/{id}.{type}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) // Parses the variable matched in the request URI. + UUID := vars["id"] + messageType := vars["type"] + + // If message id provided is blank, the message cannot exist. + if UUID == "" { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + + // Get a io.Reader instance of the message data from either the database or file system, whichever is set. + reader, err := MailGetMessageData(UUID) + // If we could not get a reader, that is more than likely due to the message not existing. + if err != nil { + http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) + return + } + // If we need to close after reading, defer the close to after this function call. + if x, ok := reader.(io.Closer); ok { + defer x.Close() + } + + // Based on response type requested, parse the message data accordingly. + if messageType == "eml" { // Original email source. + // Provide standard email message mime type. + w.Header().Set("Content-Type", "message/rfc822") + // Copy message data to response writer. + _, err := io.Copy(w, reader) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + } else if messageType == "txt" { // Plain text format. + // Provide plain text mime type. + w.Header().Set("Content-Type", "text/plain") + // Parse the email fields. + email, err := parsemail.Parse(reader) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + + // If email transfer encoding is quoted-printable, we need to decode the message. + encoding := email.Header.Get("Content-Transfer-Encoding") + if encoding == "quoted-printable" { + // Create a reader with the text body of the email. + bodyR := strings.NewReader(email.TextBody) + quotedR := quotedprintable.NewReader(bodyR) + // Copy from the reader to the response writer. + _, err = io.Copy(w, quotedR) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + } else if encoding == "base64" { // If encoded with Base64 + // Create a reader with the text body of the email. + bodyR := strings.NewReader(email.TextBody) + base64R := base64.NewDecoder(base64.StdEncoding, bodyR) + // Copy from the reader to the response writer. + _, err = io.Copy(w, base64R) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + } else { + // As this is a standard plain text email, we can just write it to the response writer. + w.Write([]byte(email.TextBody)) + } + } else if messageType == "html" { // HTML body requested. + // Set mime type to html. + w.Header().Set("Content-Type", "text/html") + // Parse the email fields. + email, err := parsemail.Parse(reader) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + + // If email transfer encoding is quoted-printable, we need to decode the message. + encoding := email.Header.Get("Content-Transfer-Encoding") + if encoding == "quoted-printable" { + // Create a reader with the html body of the email. + bodyR := strings.NewReader(email.HTMLBody) + quotedR := quotedprintable.NewReader(bodyR) + // Copy from the reader to the response writer. + _, err = io.Copy(w, quotedR) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + } else if encoding == "base64" { // If encoded with Base64 + // Create a reader with the html body of the email. + bodyR := strings.NewReader(email.HTMLBody) + base64R := base64.NewDecoder(base64.StdEncoding, bodyR) + // Copy from the reader to the response writer. + _, err = io.Copy(w, base64R) + if err != nil { // If error, return to client an error. + s.APISendGeneralResp(w, APIERR, APIReadMessage) + return + } + } else { + // Standard HTML email body, we can just write it to the response writer. + w.Write([]byte(email.HTMLBody)) + } + } else { + // No matching message type was found. Just provide a no endpoint response. + s.APISendGeneralResp(w, APIERR, APINoEndpoint) + } + }) + + // Spam reporting request must be called with a PUT request as an extra procaution to ensure we actually want to report. + api.HandleFunc("/message/{id}/learn_{type}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) // Parses the variable matched in the request URI. + UUID := vars["id"] + + // If message id provided is blank, the message cannot exist. + if UUID == "" { + s.APISendGeneralResp(w, APIERR, APINoMessage) + return + } + + // The response type should only be spam or ham. + reportType := vars["type"] + var reportingURI string + if reportType == "spam" { + reportingURI = app.config.SpamReportingSpamURI + } else if reportType == "ham" { + reportingURI = app.config.SpamReportingHamURI + } + + // If the reporting URI was not sent, we return a no endpoint found message. + if reportingURI == "" { + s.APISendGeneralResp(w, APIERR, APINoEndpoint) + return + } + + // To report spam, we need the message data. So we will find it in either the database or file system accordingly. + reader, err := MailGetMessageData(UUID) + if err != nil { // If no message found, we tell the client. + s.APISendGeneralResp(w, APIERR, APINoMessage) + return + } + // If we need to close after reading, defer the close to after this function call. + if x, ok := reader.(io.Closer); ok { + defer x.Close() + } + + // Build a response for the request. + resp := APISpamReportResp{} + anySuccess := false // We rely on this variable to determine if a successful request was made. + + // Multipart Form Data writer/buffer for sending message via post. + var b bytes.Buffer + mw := multipart.NewWriter(&b) + + // file form entry. + fw, err := mw.CreateFormFile(app.config.SpamReportingUploadName, UUID+".eml") + if err != nil { + s.APISendGeneralResp(w, APIERR, "Unable to build report.") + return + } + // Copy message data to multipart form entry. + if _, err = io.Copy(fw, reader); err != nil { + s.APISendGeneralResp(w, APIERR, "Unable to build report.") + return + } + + // If an authentication field is set, add it to the form data. + if app.config.SpamReportingAuthKey != "" { + mw.WriteField(app.config.SpamReportingAuthKey, app.config.SpamReportingAuthValue) + } + + // We are done adding to the multipart form. + mw.Close() + + // Go through the configured spam reporting URLs and submit. + for _, baseURL := range app.config.SpamReportingAPIBaseURLS { + url := baseURL + reportingURI + + // Request string map to provide feedback via API on what happend per reporting URL. + request := make(map[string]string) + request["url"] = url + + // Make request for the report using the form data. + req, err := http.NewRequest("POST", url, &b) + if err != nil { // If failed, just mark this request as failed and continue. + request["success"] = "false" + request["error"] = fmt.Sprintf("%v", err) + resp.Requests = append(resp.Requests, request) + continue + } + + // Set the content type header to the multipart formdata header with proper boundary. + req.Header.Set("Content-Type", mw.FormDataContentType()) + + // If an authentication header is set in config, we need to send it. + authHeaderS := strings.Split(app.config.SpamReportingAuthHeader, ": ") + if len(authHeaderS) == 2 { + req.Header.Set(authHeaderS[0], authHeaderS[1]) + } + + // Setup http client. + client := &http.Client{ + Timeout: time.Second * 10, + } + // Send report. + res, err := client.Do(req) + if err != nil { // If error, just store that this errored and continue. + request["success"] = "false" + request["error"] = fmt.Sprintf("%v", err) + resp.Requests = append(resp.Requests, request) + continue + } + + // If the status code is not ok, something went wrong... Store that this failed and continue. + if res.StatusCode != http.StatusOK { + request["success"] = "false" + request["error"] = fmt.Sprintf("%v", res.StatusCode) + resp.Requests = append(resp.Requests, request) + continue + } + // If we made this this far, sending the report was a success. + request["success"] = "true" + + // read the respons ebody. + defer res.Body.Close() + body, _ := ioutil.ReadAll(res.Body) + request["response"] = string(body) + // Add the request to the list of requests in the response. + resp.Requests = append(resp.Requests, request) + // A request was successful. + anySuccess = true + } + // If no successful request, we return an erro. + if !anySuccess { + resp.Status = APIERR + resp.Error = "No successful request was made." + } else { // If a request was successful, return an ok. + resp.Status = APIOK + } + // Send response. + s.JSONResponse(w, resp) + }).Methods("PUT") // Adds requirement of PUT method to the spam reporter request. + + // Pull message entry. + api.HandleFunc("/message/{id}", func(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) // Parses the variable matched in the request URI. + UUID := vars["id"] + + // The response structure. + resp := APIMessageEntryResp{} + + // If message id provided is blank, the message cannot exist. + if UUID == "" { + resp.Status = APIERR + resp.Error = APINoMessage + s.JSONResponse(w, resp) + return + } + + // Search the database for a message log entry for the message id to ensure that it exists. + var messageEntry MessageLog + app.db.Where("uuid = ?", UUID).First(&messageEntry) + // If return UUID is blank, we didn't find an entry. + if messageEntry.UUID == "" { + resp.Status = APIERR + resp.Error = APINoMessage + s.JSONResponse(w, resp) + return + } + + resp.Status = APIOK + resp.Messages = messageEntry + s.JSONResponse(w, resp) + }) + + // If nothing else, we return a not found response. + api.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.APISendGeneralResp(w, APIERR, APINoEndpoint) + }) +} diff --git a/config.go b/config.go new file mode 100644 index 0000000..22a710a --- /dev/null +++ b/config.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" + "log" + "os" + "os/user" + "path/filepath" + "time" + + "github.com/jinzhu/configor" + "github.com/urfave/cli" +) + +// Configuration Structure. +type Config struct { + HTTPBindAddr string `default:"" json:"http_bind_addr"` + HTTPPort uint `default:"80" json:"http_port"` + HTTPDebug bool `default:"false" json:"http_debug"` + SMTPBindAddr string `default:"" json:"smtp_bind_addr"` + SMTPPort uint `default:"25" json:"smtp_port"` + SMTPDomain string `default:"localhost" json:"smtp_domain"` + SysLogBindAddr string `default:"" json:"syslog_bind_addr"` + SysLogPort uint `default:"514" json:"syslog_port"` + SysLogUDP bool `default:"true" json:"syslog_udp"` + SysLogTCP bool `default:"false" json:"syslog_tcp"` + // There are some syslog ids which you may want to ignore because they belong to + // the original receiving message which is to be sent out, or because + // they belong to the message which is sent to this mail archive tool. + // This configuration allows you to specify strings which identify the syslog ids + // you wish to ignore. + // The ignore list only pertains to messages sent on or after the message-id message is received. + SysLogIgnoreContaining []string `json:"syslog_ignore_containing"` + + StaticContentPath string `default:"./www/" json:"static_content_path"` + + DBType string `default:"sqlite3" json:"database_type"` // Review documentation at http://gorm.io/docs/connecting_to_the_database.html + DBConnection string `default:"MailArchive.db" json:"database_connection"` + DBDebug bool `default:"false" json:"database_debug"` + + MailPath string `default:"db" json:"mail_path"` + + MaxAge time.Duration `default:"1209600" json:"max_age"` // Used for cleanup of old messages. Default is 2 weeks. + MaxMessageSize int `default:"5242880" json:"max_message_size"` // Default of 5 MB + + MessagesPerPage int `default:"100" json:"messages_per_page"` + + SpamReportingAPIBaseURLS []string `json:"spam_reporting_api_base_urls"` + SpamReportingAuthHeader string `json:"spam_reporting_auth_header"` // If your spam reporting tool uses a header for authentication. In `Header: Value` format + SpamReportingAuthKey string `json:"spam_reporting_auth_key"` // If your spam reporting tool uses a post variable. + SpamReportingAuthValue string `json:"spam_reporting_auth_value"` // If your spam reporting tool uses a post variable. + SpamReportingUploadName string `json:"spam_reporting_upload_name"` + SpamReportingSpamURI string `default:"learn_spam" json:"spam_reporting_spam_uri"` + SpamReportingHamURI string `default:"learn_ham" json:"spam_reporting_ham_uri"` + + UICustomBrand string `defualt:"Mail Archive" json:"ui_custom_brand"` + UIDisableSpamReporting bool `defualt:"false" json:"ui_disable_spam_reporting"` + UIDisableLogs bool `defualt:"false" json:"ui_disable_logs"` +} + +// Load the configuration. +func initConfig(c *cli.Context) Config { + usr, err := user.Current() + if err != nil { + log.Fatal(err) + } + + // Configuration paths. + localConfig, _ := filepath.Abs("./config.json") + homeDirConfig := usr.HomeDir + "/.config/mail-archive/config.json" + etcConfig := "/etc/mail-archive/config.json" + + // Determine which configuration to use. + var configFile string + if _, err := os.Stat(c.String("config")); err == nil { + configFile = c.String("config") + } else if _, err := os.Stat(localConfig); err == nil { + configFile = localConfig + } else if _, err := os.Stat(homeDirConfig); err == nil { + configFile = homeDirConfig + } else if _, err := os.Stat(etcConfig); err == nil { + configFile = etcConfig + } else { + log.Fatal("Unable to find a configuration file.") + } + + // Load the configuration file. + config := Config{} + err = configor.Load(&config, configFile) + if config.HTTPPort == 0 { + fmt.Println(err) + log.Fatal("Unable to load the configuration file.") + } + return config +} + +// Flags for the server command. +func configTestFlags() []cli.Flag { + return []cli.Flag{} +} diff --git a/database.go b/database.go new file mode 100644 index 0000000..63655dc --- /dev/null +++ b/database.go @@ -0,0 +1,60 @@ +package main + +import ( + "time" + + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +// Main message metadata storage. +type MessageLog struct { + UUID string `gorm:"primary_key" json:"uuid"` + MessageID string `json:"message_id"` + From string `json:"from"` + To string `json:"to"` + Subject string `json:"subject"` + PlainText bool `json:"plain_text"` + HTML bool `json:"html"` + Attachments bool `json:"attachments"` + SpamScore int `json:"spam_score"` + SourceIP string `default:"" json:"source_ip"` + Size int `json:"size"` + Received time.Time `json:"received"` + Status string `json:"status"` +} + +// Database storage of message data. +type Messages struct { + UUID string `gorm:"primary_key"` + Message []byte +} + +// Syslog message storage. +type SysLogMessage struct { + ID int64 `gorm:"primary_key"` + Hostname string + Timestamp time.Time + Tag string + SID string + Content string +} + +// Map of syslog message ids to email message ids with information on email status. +type SysLogIDInfo struct { + ID int64 `gorm:"primary_key"` + Hostname string + SID string + MessageID string + Status string + Ignore bool +} + +// Configure the database and add tables/adjust tables to match structures above. +func initDB(db *gorm.DB) { + db.LogMode(app.config.DBDebug) + db.AutoMigrate(&MessageLog{}) + db.AutoMigrate(&Messages{}) + db.AutoMigrate(&SysLogMessage{}) + db.AutoMigrate(&SysLogIDInfo{}) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b4a074b --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/grmrgecko/mail-archive + +go 1.14 + +require ( + github.com/DusanKasan/parsemail v1.2.0 + github.com/emersion/go-smtp v0.13.0 + github.com/google/uuid v1.1.1 + github.com/gorilla/mux v1.7.4 + github.com/gorilla/websocket v1.4.2 + github.com/jinzhu/configor v1.2.0 + github.com/jinzhu/gorm v1.9.14 + github.com/urfave/cli v1.22.4 + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + gopkg.in/mcuadros/go-syslog.v2 v2.3.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7792d9d --- /dev/null +++ b/go.sum @@ -0,0 +1,58 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DusanKasan/parsemail v1.2.0 h1:CrzTL1nuPLxB41aO4zE/Tzc9GVD8jjifUftlbTKQQl4= +github.com/DusanKasan/parsemail v1.2.0/go.mod h1:B9lfMbpVe4DMqPImAOCGti7KEwasnRTrKKn66iQefVs= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 h1:OJyUGMJTzHTd1XQp98QTaHernxMYzRaOasRir9hUlFQ= +github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= +github.com/emersion/go-smtp v0.13.0 h1:aC3Kc21TdfvXnuJXCQXuhnDXUldhc12qME/S7Y3Y94g= +github.com/emersion/go-smtp v0.13.0/go.mod h1:qm27SGYgoIPRot6ubfQ/GpiPy/g3PaZAVRxiO/sDUgQ= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/jinzhu/configor v1.2.0 h1:u78Jsrxw2+3sGbGMgpY64ObKU4xWCNmNRJIjGVqxYQA= +github.com/jinzhu/configor v1.2.0/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= +github.com/jinzhu/gorm v1.9.14 h1:Kg3ShyTPcM6nzVo148fRrcMO6MNKuqtOUwnzqMgVniM= +github.com/jinzhu/gorm v1.9.14/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD79kk= +gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/http.go b/http.go new file mode 100644 index 0000000..f01c932 --- /dev/null +++ b/http.go @@ -0,0 +1,61 @@ +package main + +import ( + "fmt" + "log" + "net/http" + + "github.com/gorilla/mux" +) + +// Basic HTTP server structure. +type HTTPServer struct { + ws *WS + wsInterface *WSInterface +} + +// This functions starts the HTTP server. +func HTTPServe() { + // Get the configuration/ + httpBindAddr := app.config.HTTPBindAddr + httpPort := app.config.HTTPPort + if app.context.String("http-bind") != "" { + httpBindAddr = app.context.String("http-bind") + } + if app.context.Uint("http-port") != 0 { + httpPort = app.context.Uint("http-port") + } + + // Create the server. + httpServer := &HTTPServer{} + app.httpServer = httpServer + // Intitialize the websocket handler. + httpServer.wsInterface = new(WSInterface) + httpServer.ws = WSInit(httpServer.wsInterface) + httpServer.wsInterface.ws = httpServer.ws + + // Set the handlers. + r := mux.NewRouter() + httpServer.RegisterAPIRoutes(r) + r.HandleFunc("/ws", httpServer.ws.Handler) + fs := http.FileServer(http.Dir(app.config.StaticContentPath)) + r.PathPrefix("/").Handler(fs) + + // The http server handler will be the mux router by default. + var handler http.Handler + handler = r + // If the debug log is enabled, we'll add a middleware handler to log then pass the request to mux router. + if app.config.HTTPDebug { + handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + log.Println(req.Method + " " + req.URL.String()) + r.ServeHTTP(w, req) + }) + } + + // Start the server. + log.Println("Starting http server on port", httpPort) + err := http.ListenAndServe(fmt.Sprintf("%s:%d", httpBindAddr, httpPort), handler) + if err != nil { + log.Fatal(err) + } +} diff --git a/mail.go b/mail.go new file mode 100644 index 0000000..22fc10f --- /dev/null +++ b/mail.go @@ -0,0 +1,252 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/DusanKasan/parsemail" + "github.com/google/uuid" +) + +// When a new message is received, this function is called to store it. +func MailSaveMessage(remoteAddr string, from string, to string, r io.Reader) error { + // We need the message body in bytes to save. + b, err := ioutil.ReadAll(r) + if err != nil { // If we can't read, we have an issue. + return err + } + // The email parser expects an io.Reader, but as we already read the reader passed. We must make a new one. + reader := bytes.NewReader(b) + // Parse the email + email, err := parsemail.Parse(reader) + if err != nil { + return err + } + // Generate a UUID for this message. + UUID := uuid.New().String() + + // Save message body using database or file if configured. + if app.config.MailPath == "db" { + // Save to database. + message := Messages{} + message.UUID = UUID + message.Message = b + app.db.Create(&message) + } else { + // If the directory configured in the config does not exist... We must fail. + if _, err := os.Stat(app.config.MailPath); err != nil { + return fmt.Errorf("Mail directory does not exist: %s", app.config.MailPath) + } + // Create the file. + messagePath := path.Join(app.config.MailPath, UUID) + fp, err := os.Create(messagePath) + if err != nil { + return err + } + // Write to the file + fp.Write(b) + fp.Close() + } + + // Create a message log entry with parsed email. + messageEntry := MessageLog{} + messageEntry.UUID = UUID + messageEntry.MessageID = email.MessageID + if len(email.From) <= 0 { + messageEntry.From = from + } else { + messageEntry.From = email.From[0].Address + } + if len(email.To) <= 0 { + messageEntry.To = to + } else { + messageEntry.To = email.To[0].Address + } + messageEntry.Subject = email.Subject + + messageEntry.PlainText = email.TextBody != "" + messageEntry.HTML = email.HTMLBody != "" + messageEntry.Attachments = len(email.Attachments) > 0 + + // If a spam level header exists, parse the score. + spamScore := email.Header.Get("X-SPAM-LEVEL") + rxScore := regexp.MustCompile("Spam detection results:\\s+([0-9]+)") + matches := rxScore.FindStringSubmatch(spamScore) + if len(matches) == 2 { + spamScoreI, err := strconv.Atoi(matches[1]) + if err != nil { + spamScoreI = 0 + } + messageEntry.SpamScore = spamScoreI + } + + // Get the source IP from the earliest received header. Default to remote address which is sending the message. + // As messages are likely forwarded to this server, the earliest received header is what we want here. + messageEntry.SourceIP = remoteAddr + // Regex to parse the received header with hostname/ip address. + rxAddr := regexp.MustCompile("from ([A-Za-z0-9-.]+) \\(.* \\[([0-9a-fA-F.:]+)\\]\\)") + // Loop through all entires of received headers. + for _, header := range email.Header["Received"] { + // Parse the header. + matches := rxAddr.FindStringSubmatch(header) + if len(matches) == 3 { + // If we got an source IP from the header, save it. + messageEntry.SourceIP = matches[1] + " (" + matches[2] + ")" + } + } + + messageEntry.Size = len(b) + messageEntry.Received = time.Now() + messageEntry.Status = "unknown" // We start as unknown and the status is updated by syslog. + + // Save the message entry. + app.db.Create(&messageEntry) + log.Printf("SMTP: Received message from %s (%d bytes)", messageEntry.From, messageEntry.Size) + + // Notify websocket subscribers of new message. + app.httpServer.wsInterface.sendMessage("receivedNewMessage", messageEntry) + + // Update message count. + app.messageCount++ + app.httpServer.wsInterface.sendMessage("updateMessageCount", app.messageCount) + return nil +} + +// Finds and outputs a reader for the message body based on UUID. +func MailGetMessageData(UUID string) (r io.Reader, err error) { + // If we are configured to use the database for storage, then we should check if the UUID is in the database. + // Otherwise, we check the path set to see if a file exists with the UUID. + if app.config.MailPath == "db" { + // Search database for message body by UUID. + var message Messages + app.db.Where("uuid = ?", UUID).First(&message) + // If not found, we provide an error. + if message.UUID == "" { + err = fmt.Errorf(APINoMessage) + return + } + // Create a reader for the message data. + r = bytes.NewReader(message.Message) + } else { + // Verify that the UUID exists in the file system. + if _, err = os.Stat(path.Join(app.config.MailPath, UUID)); err != nil { + return + } + // If the file exists, we open it to return. + r, err = os.Open(path.Join(app.config.MailPath, UUID)) + } + // Return reader. + return +} + +// To try and make the syslog code light weight, this function was created +// to update the status of messages to what was parsed in the syslog. +// This function will read an update queue map of syslog ids with updated statuses. +func RunSysLogMailUpdateQueue() { + ticker := time.NewTicker(5 * time.Second) + for _ = range ticker.C { // Every 5 seconds. + // We want to keep track as to rather statuses were updated to notify subscribers. + updated := false + + // Copy the update queue so that we can empty the main update queue. + updateQueue := make(map[string]bool) + for key, val := range app.sysLogMailUpdateQueue { + updateQueue[key] = val + } + // We empty the main update queue as the syslog may have status changes during this run. + // We want to ensure that those changes do not get lost. + app.sysLogMailUpdateQueue = make(map[string]bool) + + // Loop the update queue. + for sidHostname, _ := range updateQueue { + // Update queue should contain syslog id and hostname separated by a colon. + // We pair the syslog id with the hostname just incase different hosts use the same syslog id. + s := strings.Split(sidHostname, ":") + // If there is more or less than 2 parts, this is invalid. + if len(s) != 2 { + continue + } + sid := s[0] + hostname := s[1] + + // Pull the syslog id information from the database. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + // If nothing was returned, or this one is set to be ignored... We will stop here. + // When a syslog id is ignored, it is likely due to it being either the main message received before sending out, + // or it is the message forwarded to Mail Archive which is not the main mail delivery status. + if match.SID == "" || match.Ignore { + continue + } + + // Pull the message log entry matching the message id associated with the syslog id. + var messageEntry MessageLog + app.db.Where("message_id = ?", match.MessageID).First(&messageEntry) + // If we found the message log entry, we can update the status to match our syslog id delivery status. + if messageEntry.UUID != "" { + messageEntry.Status = match.Status + app.db.Save(&messageEntry) + // As we updated a message log entry, we want to inform the subscribers that an update occurred. + updated = true + } + } + // If we updated the status of a message log entry, we need to inform subscribers connected to websocket. + if updated { + app.httpServer.wsInterface.sendMessage("messageStatusesUpdated", true) + } + } +} + +// This function will run a database cleanup of old messages every 30 minutes. +func RunDatabaseCleanup() { + ticker := time.NewTicker(30 * time.Minute) + for _ = range ticker.C { + // Get the oldest date we will allow at this point in time based on the configured maximum age. + maxAge := time.Now().Add(app.config.MaxAge * time.Second * -1) + + // We want to just pull UUID and message id of the old messages to be cleaned up. + type MessageIDs struct { + UUID string + MessageID string + } + var messageIDs []MessageIDs + app.db.Table("message_logs").Select("uuid,message_id").Where("received <= ?", maxAge).Scan(&messageIDs) + + // Loop through all found old messages to clean up the database. + for _, message := range messageIDs { + // Find syslog id information entries matching this message. + var matches []SysLogIDInfo + app.db.Where("message_id = ?", message.MessageID).Find(&matches) + // With each found syslog id, we need to delete the syslog messages and the syslog id information. + for _, match := range matches { + app.db.Where("s_id = ? AND hostname = ?", match.SID, match.Hostname).Delete(SysLogMessage{}) + app.db.Delete(&match) + } + + // Delete the message log entry for this message. + app.db.Where("uuid = ?", message.UUID).Delete(MessageLog{}) + // Delete message data matching the UUID for the message. + app.db.Where("uuid = ?", message.UUID).Delete(Messages{}) + // If the configured mail storage path is not the database, remove it from the file system. + if app.config.MailPath != "db" { + if _, err := os.Stat(path.Join(app.config.MailPath, message.UUID)); err == nil { + os.Remove(path.Join(app.config.MailPath, message.UUID)) + } + } + // Update message count. + app.messageCount-- + } + + // Send updated message count. + app.httpServer.wsInterface.sendMessage("updateMessageCount", app.messageCount) + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..fe39420 --- /dev/null +++ b/main.go @@ -0,0 +1,87 @@ +package main + +import ( + "log" + "os" + + "github.com/emersion/go-smtp" + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/mysql" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" + "github.com/urfave/cli" + "gopkg.in/mcuadros/go-syslog.v2" +) + +// Global application structure for communicating between servers and storing information. +type App struct { + context *cli.Context + config Config + db *gorm.DB + httpServer *HTTPServer + smtpServer *smtp.Server + sysLogServer *syslog.Server + sysLogMailUpdateQueue map[string]bool + messageCount uint +} + +var app *App + +// Main start of the application. +func appInit(c *cli.Context) { + app = new(App) + app.context = c + app.config = initConfig(c) + + // Connect to the database. + db, err := gorm.Open(app.config.DBType, app.config.DBConnection) + if err != nil { + log.Fatal(err) + } + initDB(db) + app.db = db + + // Get message count. + db.Model(&MessageLog{}).Count(&app.messageCount) + + // Automatically clean up old email every 30 minutes. + go RunDatabaseCleanup() + + // Start SysLog servers. + app.sysLogMailUpdateQueue = make(map[string]bool) // Must initialize maps. + go SysLogServe() + // As syslog updates email status, we need to also update related messages. + go RunSysLogMailUpdateQueue() + + // Start SNMTP server. + go SMTPServe() + HTTPServe() +} + +func main() { + capp := cli.NewApp() + capp.Name = "mail-archive" + capp.Usage = "Email Archive Server with SysLog support and web interface." + capp.EnableBashCompletion = true + capp.Version = "0.1" + capp.Action = appInit // By default, we start the initialize function. + + capp.Flags = []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Usage: "Load configuration from `FILE`", + }, + cli.StringFlag{Name: "http-bind"}, + cli.UintFlag{Name: "http-port"}, + cli.StringFlag{Name: "smtp-bind"}, + cli.UintFlag{Name: "smtp-port"}, + cli.StringFlag{Name: "smtp-domain"}, + cli.StringFlag{Name: "syslog-bind"}, + cli.UintFlag{Name: "syslog-port"}, + } + + err := capp.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} diff --git a/smtp.go b/smtp.go new file mode 100644 index 0000000..64de2c0 --- /dev/null +++ b/smtp.go @@ -0,0 +1,107 @@ +package main + +import ( + "fmt" + "io" + "log" + "net" + "time" + + "github.com/emersion/go-smtp" +) + +// The backend structure is called for authentication of a new session. +type SMTPBackend struct { + smtp.Backend +} + +// During the process of receiving an email, this session is called. +type SMTPSession struct { + smtp.Session + remoteAddr net.Addr + from string + to string +} + +// On login, we do not care about authentication. So we just start a new session and provide it ;) +func (b *SMTPBackend) Login(state *smtp.ConnectionState, username, password string) (smtp.Session, error) { + return &SMTPSession{ + remoteAddr: state.RemoteAddr, + }, nil +} + +// We want to receive all emails, including anonymous emails. +func (b *SMTPBackend) AnonymousLogin(state *smtp.ConnectionState) (smtp.Session, error) { + return &SMTPSession{ + remoteAddr: state.RemoteAddr, + }, nil //return nil, smtp.ErrAuthRequired +} + +// The session has provided mail options and who the message is from. +func (s *SMTPSession) Mail(from string, opts smtp.MailOptions) error { + // Store the from in the session for when we receive the final message data. + s.from = from + return nil +} + +// The session has provided who the mail is to. +func (s *SMTPSession) Rcpt(to string) error { + // Store who the mail is to for final message data. + s.to = to + return nil +} + +// The session has provided the data for the message. +func (s *SMTPSession) Data(r io.Reader) error { + // Save the message to the database. + err := MailSaveMessage(s.remoteAddr.String(), s.from, s.to, r) + if err != nil { + log.Println("Unable to parse email:", err) + } + return nil +} + +// When the SMTP session is requested to start over. +func (s *SMTPSession) Reset() {} + +// When the session is done completely. +func (s *SMTPSession) Logout() error { + return nil +} + +// This function starts the SMTP server. +func SMTPServe() { + // Get the configuration. + smtpBindAddr := app.config.SMTPBindAddr + smtpPort := app.config.SMTPPort + smtpDomain := app.config.SMTPDomain + if app.context.String("smtp-bind") != "" { + smtpBindAddr = app.context.String("smtp-bind") + } + if app.context.Uint("smtp-port") != 0 { + smtpPort = app.context.Uint("smtp-port") + } + if app.context.String("smtp-domain") != "" { + smtpDomain = app.context.String("smtp-domain") + } + + // Create the SMTP server with our custom backend. + smtpBackend := &SMTPBackend{} + smtpServer := smtp.NewServer(smtpBackend) + app.smtpServer = smtpServer + + // Configure the SMTP server. + smtpServer.Addr = fmt.Sprintf("%s:%d", smtpBindAddr, smtpPort) + smtpServer.Domain = smtpDomain + smtpServer.ReadTimeout = 10 * time.Second + smtpServer.WriteTimeout = 10 * time.Second + smtpServer.MaxMessageBytes = app.config.MaxMessageSize + smtpServer.MaxRecipients = 50 + smtpServer.AllowInsecureAuth = true + + // Start the server. + log.Println("Starting smtp server on port", smtpPort) + if err := smtpServer.ListenAndServe(); err != nil { + log.Fatal(err) + } +} diff --git a/syslog.go b/syslog.go new file mode 100644 index 0000000..87a865a --- /dev/null +++ b/syslog.go @@ -0,0 +1,299 @@ +package main + +import ( + "fmt" + "log" + "regexp" + "strings" + "time" + + "gopkg.in/mcuadros/go-syslog.v2" +) + +// Message connection information for keeping track of when a +// disconnection happens and properly associate the log messages. +type SysLogMailConnection struct { + sourceAddr string + sid string + started time.Time + shouldIgnore bool +} + +// Buffer of log messages not yet associated with an email queue id (syslog id) +// and information of connections made with their associated syslog id. +type SysLogBuffer struct { + sourceAddr string + logMessages []map[string]interface{} + activeConnections []SysLogMailConnection +} + +var sysLogBuffer *SysLogBuffer + +// If the buffer has a new connection, we associate it with the next received syslog id. +func SysLogCheckIfNewConnection(sid string) { + if sysLogBuffer.sourceAddr != "" { + // Save the new connection to the active connections buffer. + newConnection := SysLogMailConnection{} + newConnection.started = time.Now() + newConnection.sourceAddr = sysLogBuffer.sourceAddr + newConnection.sid = sid + sysLogBuffer.activeConnections = append(sysLogBuffer.activeConnections, newConnection) + + // Any log messages buffered for the new connection are now associated to this syslog id. + for _, logMessage := range sysLogBuffer.logMessages { + SysLogStoreMessage(logMessage, sid) + } + + // We can reset the buffer for the next new connection. + sysLogBuffer.logMessages = nil + sysLogBuffer.sourceAddr = "" + } +} + +// Store a syslog message to the database. +func SysLogStoreMessage(logMessage map[string]interface{}, sid string) { + content := logMessage["content"].(string) + hostname := logMessage["hostname"].(string) + + // Check to see if the message received matches any ignore strings set. + for _, ignore := range app.config.SysLogIgnoreContaining { + if strings.Contains(content, ignore) { + // If we match, look in the database for syslog id information entries. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + if match.SID != "" { + // If we found the syslog id information, we can update it to ignore. + match.Ignore = true + app.db.Save(&match) + } + } + } + + // Below we check to see if the message changes the delivery status of the message. + if strings.Contains(content, "quarantine") { + // If this message queue id was quarantined, find the database entry and update. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + if match.SID != "" { + match.Status = "quarantined" + // As the queue id could be the original message which could be ignored, + // we want to not ignore this message as a quarantine means another message queue id + // was never generated for the delivery of the message. + match.Ignore = false + app.db.Save(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[sid+":"+hostname] = true + } + } else if strings.Contains(content, "status=sent") { + // If this message queue id was sent, we update the database. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + if match.SID != "" { + match.Status = "sent" + app.db.Save(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[sid+":"+hostname] = true + } + } else if strings.Contains(content, "250 2.5.0 OK") { + // If this message queue id was sent, we update the database. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + // For 250 status, only update if not quarantined. + if match.SID != "" && match.Status != "quarantined" { + match.Status = "sent" + app.db.Save(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[sid+":"+hostname] = true + } + } else if strings.Contains(content, "status=deferred") { + // If this message queue id was deferred, we update the database. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + if match.SID != "" { + match.Status = "deferred" + app.db.Save(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[sid+":"+hostname] = true + } + } else if strings.Contains(content, "status=bounced") { + // If this message queue id was bounced, we update the database. + var match SysLogIDInfo + app.db.Where("s_id = ? AND hostname = ?", sid, hostname).First(&match) + if match.SID != "" { + match.Status = "bounced" + app.db.Save(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[sid+":"+hostname] = true + } + } + + // Save the message to the database. + log := SysLogMessage{} + log.Hostname = hostname + log.Timestamp = logMessage["timestamp"].(time.Time) + log.Tag = logMessage["tag"].(string) + log.SID = sid + log.Content = content + app.db.Create(&log) +} + +// As the syslog server sends messages, we parse them. +func SysLogRunner(channel syslog.LogPartsChannel) { + // Below are all regular expressions used to match message data. + + // Daemon tags we accept messages from. + rxMailMessage := regexp.MustCompile("(?i)postfix|exim|smtp-filter") + // New connection messages. + rxConnection := regexp.MustCompile("^connect from (.*\\[[0-9A-Fa-f:.]+\\])") + // SMTP disconnection message. + rxDisconnect := regexp.MustCompile("^disconnect from (.*\\[[0-9A-Fa-f:.]+\\])") + // Message queue id. + rxMailID := regexp.MustCompile("^([A-Za-z0-9]+): ") + // End of message ok with message queue id. + rxMailIDOk := regexp.MustCompile("OK \\(([A-Za-z0-9]+)\\)") + // A message queue id association with message id header. + rxMailMessageID := regexp.MustCompile("^([A-Za-z0-9]+):.*message-id=<(.*)>") + + // When a log message is received. + for logParts := range channel { + // Check to see if the received tag is one associated with emails. + tag := logParts["tag"].(string) + if !rxMailMessage.MatchString(tag) { + continue + } + content := logParts["content"].(string) + + // Parse content to see if a message id association is being provided. + matches := rxMailMessageID.FindStringSubmatch(content) + if len(matches) == 3 { + // If we received a message id association with the queue id, + // add it to the database for syslog id information. + match := SysLogIDInfo{} + match.Hostname = logParts["hostname"].(string) + match.SID = matches[1] + match.MessageID = matches[2] + match.Status = "queued" + app.db.Create(&match) + log.Println("Syslog:", match.SID, match.Status) + // The status was updated, so we can save to the queue for procoessing. + app.sysLogMailUpdateQueue[match.SID+":"+match.Hostname] = true + + // Check if this is the first message queue id received for the connection. + SysLogCheckIfNewConnection(matches[1]) + // Save this message to the syslog database. + SysLogStoreMessage(logParts, matches[1]) + continue + } + + // If message contains a queue id, we can just store it. Ignore NOQUEUE messages. + matches = rxMailID.FindStringSubmatch(content) + if len(matches) == 2 && matches[1] != "NOQUEUE" { + // Check if this is the first message queue id received for the connection. + SysLogCheckIfNewConnection(matches[1]) + // Save this message to the syslog database. + SysLogStoreMessage(logParts, matches[1]) + continue + } + + // If this is a end of message ok message, we can store it. + matches = rxMailIDOk.FindStringSubmatch(content) + if len(matches) == 2 { + // Check if this is the first message queue id received for the connection. + SysLogCheckIfNewConnection(matches[1]) + // Save this message to the syslog database. + SysLogStoreMessage(logParts, matches[1]) + continue + } + + // If this is a new connection message, update the buffer. + matches = rxConnection.FindStringSubmatch(content) + if len(matches) == 2 { + // Save source address. + sysLogBuffer.sourceAddr = matches[1] + // Reset and store current message in message buffer. + sysLogBuffer.logMessages = nil + sysLogBuffer.logMessages = append(sysLogBuffer.logMessages, logParts) + continue + } + + // If this is a disconnection message, we can check if it matches an existing connection and close ito ut. + matches = rxDisconnect.FindStringSubmatch(content) + if len(matches) == 2 { + // Go through the active connections. + for i := 0; i < len(sysLogBuffer.activeConnections); i++ { + connection := sysLogBuffer.activeConnections[i] + // If this connection is older than 1 minute... We can discard it. + if time.Since(connection.started).Seconds() >= 60 { + // Discard this connection. + sysLogBuffer.activeConnections = append(sysLogBuffer.activeConnections[:i], sysLogBuffer.activeConnections[i+1:]...) + i-- + } else if matches[1] == connection.sourceAddr { + // If this connection matches our disconnection, we can log the message. + SysLogStoreMessage(logParts, connection.sid) + + // We can now discard this message. + sysLogBuffer.activeConnections = append(sysLogBuffer.activeConnections[:i], sysLogBuffer.activeConnections[i+1:]...) + i-- + } + } + continue + } + // If there is a new connection, and this log message was not matched above. + // We then store this message in the buffer for associating with a message queue id above. + if sysLogBuffer.sourceAddr != "" { + sysLogBuffer.logMessages = append(sysLogBuffer.logMessages, logParts) + } + } +} + +// This functions tarts the syslog server. +func SysLogServe() { + // If syslog is not enabled, stop here. + if !app.config.SysLogUDP && !app.config.SysLogTCP { + return + } + + // Create a new syslog buffer. + sysLogBuffer = new(SysLogBuffer) + + // Get the configuration/ + sysLogBindAddr := app.config.SysLogBindAddr + sysLogPort := app.config.SysLogPort + if app.context.String("syslog-bind") != "" { + sysLogBindAddr = app.context.String("syslog-bind") + } + if app.context.Uint("syslog-port") != 0 { + sysLogPort = app.context.Uint("syslog-port") + } + + // Create the syslog server and message channel. + channel := make(syslog.LogPartsChannel) + handler := syslog.NewChannelHandler(channel) + server := syslog.NewServer() + app.sysLogServer = server + + // Configure the syslog server. + server.SetFormat(syslog.RFC3164) + server.SetHandler(handler) + if app.config.SysLogUDP { + server.ListenUDP(fmt.Sprintf("%s:%d", sysLogBindAddr, sysLogPort)) + } + if app.config.SysLogTCP { + server.ListenTCP(fmt.Sprintf("%s:%d", sysLogBindAddr, sysLogPort)) + } + + // Start the syslog server. + log.Println("Starting system log server on port", sysLogPort) + server.Boot() + + // Start the message queue channel reader. + go SysLogRunner(channel) + + // Wait until the syslog server stops. + server.Wait() +} diff --git a/websocket.go b/websocket.go new file mode 100644 index 0000000..cac8248 --- /dev/null +++ b/websocket.go @@ -0,0 +1,187 @@ +package main + +import ( + "log" + "net/http" + "time" + + "github.com/gorilla/websocket" +) + +const ( + // Time allowed to write a message to the peer. + writeWait = 10 * time.Second + + // Time allowed to read the next pong message from the peer. + pongWait = 60 * time.Second + + // Send pings to peer with this period. Must be less than pongWait. + pingPeriod = (pongWait * 9) / 10 + + // Maximum message size allowed from peer. + maxMessageSize = 1024 * 1024 +) + +// Setup an upgrader with our configuration. +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +type WSMessageInterface interface { + handleMessage(message []byte, c *WSClient) +} + +// The websocket server structure. +type WS struct { + clients map[*WSClient]bool + message chan []byte + register chan *WSClient + unregister chan *WSClient + messageInterface WSMessageInterface +} + +// Setup the websocket +func WSInit(messageInterface WSMessageInterface) *WS { + ws := &WS{ + message: make(chan []byte), + register: make(chan *WSClient), + unregister: make(chan *WSClient), + clients: make(map[*WSClient]bool), + messageInterface: messageInterface, + } + go ws.run() + return ws +} + +// Main websocket channel handler. +func (ws *WS) run() { + for { + select { + case client := <-ws.register: + // If we have a new client, add it to the client map. + ws.clients[client] = true + case client := <-ws.unregister: + // If a client is unregistering and in the client map, remove it. + if _, ok := ws.clients[client]; ok { + delete(ws.clients, client) + close(client.send) + } + case message := <-ws.message: + // A message is being sent to all clients. + for client := range ws.clients { + // Send message to client if possible. + select { + case client.send <- message: + default: + // If we were unable to send, the client is no longer connected. + close(client.send) + delete(ws.clients, client) + } + } + } + } +} + +// Websocket client structure. +type WSClient struct { + ws *WS + + // The websocket connection. + conn *websocket.Conn + + // Buffered channel of outbound messages. + send chan []byte +} + +// Read messages from the client. +func (c *WSClient) reader() { + // When we are unable to read from the client, we need to unregister and close the connection. + defer func() { + c.ws.unregister <- c + c.conn.Close() + }() + + // Set the max size of a message from the client. + c.conn.SetReadLimit(maxMessageSize) + // The client must send us keep alive messages, otherwise the connection is dead. + c.conn.SetReadDeadline(time.Now().Add(pongWait)) + // When we receive a pong from the client, the keep alive timeout is extended. + c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil }) + // Until connection is close, read messages fromt eh client. + for { + _, message, err := c.conn.ReadMessage() + // If we received an error, something is wrong and we need to close the connection. + if err != nil { + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("error: %v", err) + } + break + } + // We got a connection, so we can pass it to the message interface for handling. + c.ws.messageInterface.handleMessage(message, c) + } +} + +// Watches the message send channel for new messages to send to the client. +func (c *WSClient) writer() { + // We need to pink the client every now and then. + ticker := time.NewTicker(pingPeriod) + // If the writer channel closes, we need to stop the ping ticker and close the connection. + defer func() { + ticker.Stop() + c.conn.Close() + }() + // Check for new message or ping ticker, whichever happens first. + for { + select { + case message, ok := <-c.send: + // If a new message was added tot he channel, we need to write it. + c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + // If the message received is closing the channel, we need to send a close message. + if !ok { + c.conn.WriteMessage(websocket.CloseMessage, []byte{}) + return + } + + // Get the next writer for a text message. + w, err := c.conn.NextWriter(websocket.TextMessage) + if err != nil { + return + } + + // Write the message to the client. + w.Write(message) + + // Close the writer. If an error, we stop the write channel. + if err := w.Close(); err != nil { + return + } + case <-ticker.C: + // Send a ping message. + c.conn.SetWriteDeadline(time.Now().Add(writeWait)) + if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// New connection handler for websockets that creates a client and upgrades the connection. +func (ws *WS) Handler(w http.ResponseWriter, r *http.Request) { + log.Println("New websocket connection from ", r.RemoteAddr) + // Upgade the connection to a websocket connection. + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Println(err) + return + } + + // Create a new client and register it. + client := &WSClient{ws: ws, conn: conn, send: make(chan []byte, 256)} + ws.register <- client + + // Start the reader and writer. + go client.reader() + go client.writer() +} diff --git a/websocket_handler.go b/websocket_handler.go new file mode 100644 index 0000000..58b3e68 --- /dev/null +++ b/websocket_handler.go @@ -0,0 +1,86 @@ +package main + +import ( + "encoding/json" + "fmt" +) + +// The websocket interface used for the mail program. +type WSInterface struct { + ws *WS +} + +// A standard message sent or recieved from the client. +type WSMessage struct { + MessageType string `json:"type"` + Message interface{} `json:"msg"` +} + +// A general websocket response. +type WSGeneralResp struct { + Status string `json:"status"` + Error string `json:"error"` +} + +func (ws *WSInterface) buildMessageJson(msgType string, msg interface{}) (message []byte, err error) { + // Build a message. + wsMessage := WSMessage{ + MessageType: msgType, + Message: msg, + } + // Encode message to json. + message, err = json.Marshal(wsMessage) + return +} + +// Send a message to the subscribed clients. +func (ws *WSInterface) sendMessage(msgType string, msg interface{}) error { + // Build json. + message, err := ws.buildMessageJson(msgType, msg) + if err != nil { + return err + } + // Send message to subscribed clients. + ws.ws.message <- message + return nil +} + +// Send a message to a specific client. +func (ws *WSInterface) sendMessageToClient(msgType string, msg interface{}, c *WSClient) error { + // Build json. + message, err := ws.buildMessageJson(msgType, msg) + if err != nil { + return err + } + // Send message to client. + c.send <- message + return nil +} + +// Handle a message from a client. +func (ws *WSInterface) handleMessage(message []byte, c *WSClient) { + // Message should be in standard json. + wsMessage := WSMessage{} + err := json.Unmarshal(message, &wsMessage) + + // If we could not parse the message, return an error. + if err != nil { + resp := WSGeneralResp{ + Status: APIERR, + Error: fmt.Sprintf("Unable to decode request %v", err), + } + ws.sendMessageToClient("error", resp, c) + return + } + + // Depending on the message type, handle the message. + switch wsMessage.MessageType { + default: + // By default, we do nothing but return an error. + resp := WSGeneralResp{ + Status: APIERR, + Error: fmt.Sprintf("No handler of type %v", wsMessage.MessageType), + } + ws.sendMessageToClient("error", resp, c) + } +} diff --git a/www/bower.json b/www/bower.json new file mode 100644 index 0000000..001ab56 --- /dev/null +++ b/www/bower.json @@ -0,0 +1,31 @@ +{ + "name": "mail-archive", + "main": "index.js", + "authors": [ + "James Coleman " + ], + "description": "Email Archive Server with SysLog support and web interface.", + "keywords": [ + "Email", + "archive", + "message", + "syslog" + ], + "license": "MIT", + "homepage": "https://mrgeckosmedia.com", + "private": true, + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ], + "dependencies": { + "bootstrap": "^4.5.0", + "jquery": "^3.5.1", + "fontawesome": "^5.14.0", + "mustache": "^4.0.1", + "moment": "^2.27.0" + } +} diff --git a/www/index.css b/www/index.css new file mode 100644 index 0000000..c05b5d2 --- /dev/null +++ b/www/index.css @@ -0,0 +1,134 @@ +#templates { + display: none; +} +#message { + width: 100%; + height: 54px; + text-align: center; + background-color: red; + color: white; + position: absolute; + top: 0; + left: 0; + font-size: 25pt; + display: none; + z-index: 10; +} +#customcss { + display: none; +} + +html, body { + padding: 0; + height: 100%; +} +.wrapper { + display: flex; + flex-flow: column; + height: 100%; + min-height: 100%; + width: 100%; +} +nav.navbar { + padding: 0.2rem 1rem; + flex: 0 1 auto; +} +#message_list_container { + width: 100%; + overflow-y: scroll; + overflow-x: hidden; + height: 10em; + min-height: 3em; + display: block; + flex: 0 1 auto; +} +#message_list { + margin-bottom: 0; +} +#message_list tr.active { + background-color: #0058CF; + color: #fff; +} +#message_list td, #message_list th { + padding: 0; + padding-left: 4px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +#message_list thead { + background-color: #eee; + position: sticky; + top: 0; +} +#message_list tbody tr { + cursor: pointer; +} +#message_list td.to, #message_list td.from, +#message_list th.to, #message_list th.from { + max-width: 300px; +} +#message_list td.status, #message_list th.status { + width: 100px; +} +#message_list td.received, #message_list th.received { + width: 172px; +} +#message_list_resizer { + cursor: ns-resize; + border-top: 1px solid #ccc; + border-bottom: 1px solid #fff; + line-height: 2px; + flex: 0 1 auto; +} +#message_view { + display: flex; + flex-flow: column; + flex: 1 1 auto; +} +#message_header { + background-color: #eee; + flex: 0 1 auto; +} +#message_header table { + font-size: 10pt; +} +#message_header table .title { + width: 70px; + text-align: right; + font-weight: bold; + color: #555; +} +#message_header table .content { + min-width: 70px; + text-align: left; +} +#message_header .nav-tabs { + padding-top: 10px; +} +#message_header .nav-item { + padding-left: 2px; + padding-right: 2px; +} +#message_header .nav-link { + color: #000; + background-color: #ddd; + padding: .2rem 1rem; +} +#message_header .nav-link.active { + color: #000; + background-color: #fff; +} +#message_header .nav-link:hover { + background-color: #eee; +} +#message_header .nav-link:disabled { + background-color: #ccc; +} +#message_header .btn-group { + float: right; + padding-right: 10px; +} +#message_contents { + overflow: scroll; +} diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..65cf90f --- /dev/null +++ b/www/index.html @@ -0,0 +1,109 @@ + + + + + + + Mail Archive + + + + + + + + + + + + + +
+
+ +
+
+
+ +
+ + + + + + + + + + + +
FromToSubjectStatusReceived
+
+
 
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
ReceivedSize
FromScore
ToStatus
SubjectSource IP
+
+ + + +
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+
+
+ diff --git a/www/index.js b/www/index.js new file mode 100644 index 0000000..c872e30 --- /dev/null +++ b/www/index.js @@ -0,0 +1,779 @@ +// Converts bytes to a human readable value. +function bytesToHuman(bytes) { + var i = Math.floor(Math.log(bytes) / Math.log(1024)), + sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + + return (bytes / Math.pow(1024, i)).toFixed(2) * 1 + ' ' + sizes[i]; +} + +// A timer for when messages are shown on screen to auto hide. +var messageTimer = null; + +// Hide the message on screen. +function hideMessage() { + // Get the message div. + var message = document.getElementById("message"); + message.style.display = "none"; // Do not display. + // Clear the message timer. + clearTimeout(messageTimer); + messageTimer = null; +} + +// Display a standard message on screen. +function displayMessage(message) { + displayMessage(message, "cadetblue", true); +} + +// Display a sucessful message with green color. +function displaySuccess(message) { + displayMessage(message, "green", true); +} + +// Display an error message with red color. +function displayError(message) { + displayMessage(message, "red", true); +} + +// Display a message with a timeout and color specified. +function displayMessage(message, color, timeout) { + // If no color defined, we use cadetblue. + if (color == undefined) { + color = "cadetblue"; + } + // Log the message to the javascript console. + console.log(message); + + // Get the message div. + var messageDiv = document.getElementById("message"); + messageDiv.innerText = message; + messageDiv.style.backgroundColor = color; + messageDiv.style.display = "block"; // Make message visable. + + // If a message timer already exists, we can clear the timeout to prevent it from hiding this message. + if (messageTimer!=null) { + clearTimeout(messageTimer); + } + // If message is to timeout, set a timeout to hide in 5 seconds. + if (timeout) { + messageTimer = setTimeout(hideMessage, 5000); + } +} + +// Configuration Options +var UIDisableSpamReporting = false; +var UIDisableLogs = false; + +// The width calculated for the subject. +var UISubjectWidth = 0; + +// Build custom CSS based on configuration. +function rebuildCustomCSS() { + var cssConfig = '"; + // Add style tag to page. + document.getElementById("customcss").innerHTML = cssConfig; +} + +// Load configuration from API. +function loadConfig() { + // Call the API. + $.ajax({ + dataType: "json", + type: "GET", + url: "/api/config" + }) + .done(function(data) { + // If an error ocurred. Display it. + if (data.status=="error") { + displayError("Unable to load configuration: "+data.error); + return; + } + + // Save configuration. + UIDisableSpamReporting = data.disable_spam_reporting; + UIDisableLogs = data.disable_logs; + + if (data.custom_brand!="") { + $("#navbar_brand").text(data.custom_brand); + } + $("#message_count").text(data.message_count.toLocaleString()); + + // Rebuild CSS with new config. + rebuildCustomCSS(); + }) + .fail(function(jqXHR, textStatus) { + // On error, display a message. + displayError("Unable to load configuration: "+textStatus); + }); +} + +// Storage of the currently selected email message. +var selectedMessage = null; + +// Auto resize global resize based variable. +// This variable is basically the current top offset of the message contents view, minus the message list height. +// This allows us to easily determine the max height of the message contents view by taking the window height +// and substracting the message height and this base height. +var messageResizeBase = 0; + +// Handle a window resize event. +function handleResize() { + // Get the current message list height from either the message list container itself, or storage. + var messagesH = $("#message_list_container").height(); + if (localStorage && 'message_list_height' in localStorage) { + messagesH = localStorage.message_list_height; + $("#message_list_container").height(messagesH); + } + // If we don't have a resize base already calculated, calculate it. + if (messageResizeBase==0) { + messageResizeBase = $("#message_contents").offset().top-messagesH; + } + // The new message contents height should be the window height minus messages list height minus the resize base. + var messageH = $(window).height()-messagesH-messageResizeBase; + $("#message_contents").height(messageH); + + // Get width of other columns. + var fromWidth = $("th.from").width(); + var toWidth = $("th.to").width(); + var statusWidth = 0; + if ($("th.status").is(":visible")) { + statusWidth = $("th.status").width(); + } + var receivedWidth = $("th.received").width(); + // Subtract width of other volumes from windows width. + UISubjectWidth = $(window).width()-(fromWidth+toWidth+statusWidth+receivedWidth+16); // 16 is padding. + // Limit to width of 100 pixels. + if (UISubjectWidth<100) { + UISubjectWidth = 100; + } + // Build the css. + rebuildCustomCSS(); +} + +// This function will check to see where the active message is in the list, and determine if it is visable. +// If the message is not visable in the message list, it will scroll to make it visable. +function scollToActiveMessageIfNeeded() { + // Get the current selection, and stop if no selection is made. + var selection = $("#message_list .active"); + if (selection.length<=0) { + return; + } + + // Determine the selection's position in the container list. + var selectionTop = selection.position().top; + var rowH = selection.height(); + var container = $("#message_list_container"); + + // If the message is above the scroll position, we need to scroll up. + if (container.scrollTop()>selectionTop-rowH) { + container.animate({ + scrollTop: selectionTop-rowH + }, 200); + } else if (container.scrollTop()+container.height()"); + iframe.css("height", "100%"); + iframe.css("width", "100%"); + iframe.attr("src", "/api/message/"+selectedMessage.uuid+extension); + + // Append iframe to the message contents view. + $("#message_contents").html(""); + $("#message_contents").append(iframe); + } else { + // All other source types are handled here. + + // Get the message source from the API. + $.ajax({ + url: "/api/message/"+selectedMessage.uuid+extension + }) + .done(function(data) { + // If an error was returned, we display it. + if (data.status!=undefined) { + displayError("Unable to pull message: "+data.error); + return; + } + + // We display plain text message contents in a pre-formatted element. + var preFormated = $("
").text(data);
+
+            // Append the pre-formatted element to the message contents.
+            $("#message_contents").html("");
+            $("#message_contents").append(preFormated);
+        })
+        .fail(function(jqXHR, textStatus) {
+            // Om error, display a message.
+            displayError("Unable to pull message: "+textStatus);
+        });
+    }
+}
+
+// This function is used to update the currently selected message view.
+function updateSelectedMessage() {
+    // Update the header information.
+    $("#message_header .received").text(selectedMessage.formatted_date);
+    $("#message_header .size").text(bytesToHuman(selectedMessage.size));
+    $("#message_header .from").text(selectedMessage.from);
+    $("#message_header .to").text(selectedMessage.to);
+    $("#message_header .subject").text(selectedMessage.subject);
+    $("#message_header .spam_score").text(selectedMessage.spam_score);
+    $("#message_header .status").text(selectedMessage.status);
+    $("#message_header .source_ip").text(selectedMessage.source_ip);
+
+    // If no plain text, this must be a html email.
+    if (!selectedMessage.plain_text) {
+        // Disable plain text source selection tab.
+        $("#message_header .nav-tabs .plaintext").prop("disabled", true);
+        // Enable the html source selection tab.
+        $("#message_header .nav-tabs .html").prop("disabled", false);
+        // If the currently selected source tab is disabled, we need to select html.
+        if ($("#message_header .nav-tabs .active").prop("disabled")) {
+            $("#message_header .nav-tabs .html").click();
+        } else {
+            // Otherwise, select the active tab.
+            $("#message_header .nav-tabs .active").click();
+        }
+    } else {
+        // When plaintext is avaiable, we need to disable HTML source selection only if there is no HTML.
+        $("#message_header .nav-tabs .html").prop("disabled", !selectedMessage.html);
+        // We can enable the plain text source selection.
+        $("#message_header .nav-tabs .plaintext").prop("disabled", false);
+        // If the currently selected source tab is disabled, we need to select plain text.
+        if ($("#message_header .nav-tabs .active").prop("disabled")) {
+            $("#message_header .nav-tabs .plaintext").click();
+        } else {
+            // Otherwise, select the active tab.
+            $("#message_header .nav-tabs .active").click();
+        }
+    }
+}
+
+// When a message is selected in the message list, this function is called.
+function handleMessageListSelection() {
+    // Get the selected message data.
+    var selection = $(this);
+    var message = JSON.parse(selection.attr("data"));
+    selectedMessage = message;
+
+    // Change the selelected message in the message list to this selection.
+    $("#message_list tr.active").removeClass("active");
+    selection.addClass("active");
+
+    // Update the location hash URI to this message.
+    window.location.hash = "uuid="+message.uuid;
+
+    // Update the selected message.
+    updateSelectedMessage();
+}
+
+// When the learn ham spam reporting button is clicked, this function is called.
+function learnHam() {
+    // If no message is selected, we display a message and stop here.
+    if (selectedMessage==null) {
+        displayMessage("Select a message first.");
+        return;
+    }
+    // Confirm that this action is actually wanted to occur.
+    var r = confirm("Are you sure you want to report as ham?");
+    if (r != true) {
+        return
+    }
+
+    // Send request to the API.
+    $.ajax({
+        dataType: "json",
+        type: 'PUT',
+        url: "/api/message/"+selectedMessage.uuid+"/learn_ham"
+    })
+    .done(function(data) {
+        // If error, display message.
+        if (data.status=="error") {
+            displayError("Unable to report ham: "+data.error);
+            return;
+        }
+        // We Successfully submitted a report.
+        displaySuccess("Successfully reported as ham.");
+    })
+    .fail(function(jqXHR, textStatus) {
+        // On error, display message.
+        displayError("Unable to report ham: "+textStatus);
+    });
+}
+
+// When the learn spam spam reporting button is clicked, this function is called.
+function learnSpam() {
+    // If no message is selected, we display a message and stop here.
+    if (selectedMessage==null) {
+        displayMessage("Select a message first.");
+        return;
+    }
+    // Confirm that this action is actually wanted to occur.
+    var r = confirm("Are you sure you want to report as spam?");
+    if (r != true) {
+        return
+    }
+
+    // Send request to the API.
+    $.ajax({
+        dataType: "json",
+        type: "PUT",
+        url: "/api/message/"+selectedMessage.uuid+"/learn_spam"
+    })
+    .done(function(data) {
+        // If error, display message.
+        if (data.status=="error") {
+            displayError("Unable to report spam: "+data.error);
+            return;
+        }
+        // We Successfully submitted a report.
+        displaySuccess("Successfully reported as spam.");
+    })
+    .fail(function(jqXHR, textStatus) {
+        // On error, display message.
+        displayError("Unable to report spam: "+textStatus);
+    });
+}
+
+// When the document has fully loaded, we get everything started.
+$(document).ready(function() {
+    // Connect to websockets if available.
+    if (!window["WebSocket"]) {
+        displayError("Your browser does not support websockets, auto refresh will only occur once every minute.");
+        return;
+    } else {
+        connectToWS();
+    }
+    // Laod the configuration from API.
+    loadConfig();
+
+    // Make the message list resizer element work.
+    makeMessageListResizable();
+
+    // On window resize events, adjust view sizes.
+    $(window).resize(handleResize);
+    // Update the view sizes.
+    setTimeout(handleResize, 200);
+
+    // Load the message list.
+    loadMessageList();
+
+    // Every 5 seconds, we need to check if we need to refresh the message list.
+    setInterval(checkIfRefreshNeeded, 5000);
+    // Every minute, we force a refresh.
+    setInterval(function() {
+        shouldRefresh = true;
+    }, 60000);
+
+    // On input in the search field, handle it.
+    $("#searchInput").on("input", handleSearchInput);
+
+    // Handle clicking on items in the message list.
+    $("#message_list").on("click", "tr", handleMessageListSelection);
+
+    // Handle global document key down events.
+    $(document).keydown(handleKeydownEvent);
+
+    // Handle clicks on source selection tabs.
+    $("#message_header .nav-tabs .nav-link").click(handleSourceSelection);
+
+    // Handle a click on the email download button.
+    $("#mailDownloadButton").click(function() {
+        // If no message selected, stop here.
+        if (selectedMessage==null) {
+            return;
+        }
+        // Setup the download path.
+        var downloadPath = "/api/message/"+selectedMessage.uuid+".eml";
+
+        // Create an link with a download file name to allow downloading without navigating away.
+        // This is done to avoid disconnection from the websocket.
+        var a = document.createElement("a");
+        a.href = downloadPath;
+        a.download = downloadPath.substr(downloadPath.lastIndexOf('/') + 1);
+
+        // Append the link, click it and remove it.
+        document.body.appendChild(a);
+        a.click();
+        document.body.removeChild(a);
+    });
+
+    // Setup handlers for spam reporting.
+    $("#mailLearnHamButton").click(learnHam);
+    $("#mailLearnSpamButton").click(learnSpam);
+
+    // Check if a message uuid was provided in the location hash.
+    var hashParams = new URLSearchParams(window.location.hash.slice(1));
+    if (hashParams.has("uuid")) {
+        // If it was, we should load that message.
+        var uuid = hashParams.get("uuid");
+        loadMessage(uuid);
+    }
+
+    // Register for when the hash location has changed.
+    $(window).bind('hashchange', function(e) {
+        // Check if the UUID is provided in the updated hash location.
+        var hashParams = new URLSearchParams(window.location.hash.slice(1));
+        if (hashParams.has("uuid")) {
+            // If it was, we can load the message.
+            var uuid = hashParams.get("uuid");
+            loadMessage(uuid);
+        }
+    });
+});