Handle Incoming POP3 with mpop App

The way emails get around from server-to-server is fascinating. On top of the TCP protocol they use other protocols like SSL-TLS, POP3, IMAP, and SMTP. There’s probably many more than that but it’s been mostly the same since the internet came out. These protocols stick around for a long time and the ISPs I’m connecting with offer all those but require TLS at this time. In the past it was all done over clear text and without encryption.

One such implementation of an email client is mpop for POP3. Over the command line it’s possible to get this tool to download to disk all the messages from an email server. Some emails servers I’ve used store the messages inside an SQL database but I think it’s equally useful to output these messages to the file system.

If you don’t have mpop yet on OSX you can use homebrew to install it brew install mpop.

./mpop-fetch

mpop --keep=on \
  --delivery=maildir,"$MPOP_MAILDIR_PATH" \
  --uidls-file="$MPOP_UIDL_PATH" \
  --host="$MPOP_HOST" \
  --port=$MPOP_PORT \
  --pipelining=auto \
  --user="$MPOP_USERNAME" \
  --tls=on \
  --tls-certcheck=off \
  --tls-starttls=off \
  --password="echo $MPOP_PASSWORD" # or --passwordeval

mpop is trying to be secure by requiring the user to get the password from a OS keychain app. That is the reason why --password is an evaluated command and not literal. I was satisfied for my experiment to pass it via clear text or environment variable. The only other non-obvious part was the --uidls-file which appears to be a way to avoid downloading the same messages if they already exist on the disk.

It was FAST. All my messages of approximately 30 MB were downloaded in seconds.

So, in this instance I wanted to store some mailbox info inside CouchDB and connect mpop with node.

The operations are all done in series or parallel but the order of events goes like this:

  1. Ensure the emailPath exists
  2. Load email accounts, pop server, port, email address
  3. Create Maildir-compliant directories cur, new, tmp for each account
  4. On a timer later on I plan to call email.downloadAllAccountEmails at a 5-15 minute interval
  5. Append log results

./email.js (a module for a larger app)

'use strict';

var fs = require('fs'),
    path = require('path'),
    child_process = require('child_process'),
    async = require('async'),
    map = require('lodash/map'),
    mkdirp = require('mkdirp'),
    spawn = child_process.spawn,

    // pattern to match non-alphabet chars in a string
    nonLetters = /\W/gi,

    // maildir compliant directories for each account
    maildirSubdirectories = ['cur', 'new', 'tmp'];

function attach(app, options) {
    var email = {},
        emailPath;

    // it needs a path to store the emails or else it cannot continue
    if (options === undefined || options.emailPath === undefined) {
        throw new Error('options.emailPath expected');
    }

    // download emails into this dir
    emailPath = options.emailPath;
    email.emailPath = options.emailPath;

    // a place in memory to store the accounts
    email.accounts = [];

    // load from couchdb the accounts data
    email.loadEmailAccounts = function (cb) {
        // make an HTTP request to CouchDB http://.../_design/email-accounts/_view/index
        app.db.view('email-accounts', 'index', function (err, view) {
            if (err !== undefined && err !== null) {
                return cb(err);
            }

            // store the results
            email.accounts = map(view.rows, 'value');

            // support running in series or parallel with async module
            cb(undefined, email.accounts);
        });
    };

    // turn something like "michael@bologna.com" to "michaelbolognacom"
    // it strips out the `nonLetters` using that regexp
    // in that form it should be more file-system friendly
    function boxName(emailAddress) {
        return emailAddress
            .replace(nonLetters, '')
            .toLowerCase();
    }

    // use mpop to download the messages
    email.downloadAccountEmails = function (account, cb) {
        var server = account.pop,
            proc,
            procOptions,
            box = boxName(account.emailAddress),
            boxPath = path.join(emailPath, box),
            uidlPath = boxPath + '.uidl',
            logPath = path.join(app.logPath, box + '.log'),
            logOptions = {flags: 'a', autoClose: true},
            logStream = fs.createWriteStream(logPath, logOptions);

        // passing this information through process arguments didn't work
        // but using an object for changing the environment did work
        procOptions = {
            env: {
                MPOP_MAILDIR_PATH: boxPath,
                MPOP_UIDL_PATH: uidlPath,
                MPOP_HOST: server.host,
                MPOP_PORT: server.port,
                MPOP_USERNAME: account.emailAddress,
                MPOP_PASSWORD: account.password
            }
        };

        function proc_end(code, signal) {
            if (code !== 0) {
                console.log('mpop-fetch email error');
                console.log('code', code);
                console.log('signal', signal);
            }

            logStream.close();
            cb();
        }

        // launch the process to download all messages
        proc = spawn(path.join(__dirname, 'mpop-fetch'), procOptions);
        proc.stdout.pipe(logStream);
        proc.stderr.pipe(logStream);
        proc.on('close', proc_end);
    };

    // loop through the emails and download with mpop in parallel
    email.downloadAllAccountEmails = function (cb) {
        var steps;

        function download(item) {
            return async.apply(email.downloadAccountEmails, item);
        }

        steps = email.accounts.map(download);
        async.parallel(steps, cb);
    };

    // create: $emailPath/$username/cur new & tmp
    // also can be done in parallel
    // mkdirp is handy
    email.createAccountMaildirs = function (account, cb) {
        var box,
            steps;

        // remove
        box = boxName(account.emailAddress);

        function maildirPath(item) {
            return path.join(emailPath, box, item);
        }

        function mkdirStep(item) {
            return async.apply(mkdirp, item);
        }

        steps = maildirSubdirectories.map(maildirPath)
            .map(mkdirStep);

        async.parallel(steps, cb);
    };

    // loop through all accounts and create their maildirs
    email.createAllAccountMaildirs = function (cb) {
        var steps;

        function maildirStep(item) {
            return async.apply(email.createAccountMaildirs, item);
        }

        steps = email.accounts.map(maildirStep);

        async.parallel(steps, cb);
    };

    app.email = email;
}

function init(app, done) {
    var email = app.email,
        emailPath = email.emailPath,
        steps;

    steps = [
        // make the main email dir if it doesn't exist
        async.apply(mkdirp, emailPath),

        // load email accounts
        email.loadEmailAccounts,

        // ensure maildirs for this account
        email.createAllAccountMaildirs
    ];

    function steps_done(err) {
        if (err !== undefined && err !== null) {
            return console.error('error loading emails', err);
        }

        done();
    }

    async.series(steps, steps_done);
}

// it is a pluggy module plugin
// https://github.com/tcrowe/pluggy
module.exports = {
    name: 'server-email',
    attach: attach,
    init: init
};

It’s complicated but in the chance you’re working on a mail system for yourself or your company it can give some ideas. I plan to use tools like mpop and related things for personal and business. The usual email clients out there are fine but I would like to do other things.

Enjoy! Best of luck on your project and next time we’ll talk about how to do a similar thing with msmtp and also incorporate mailparser.

Author Tony Crowe, Salt Lake City, UT
top
home
Attribution
Lato font by Łukasz Dziedzicwww.latofonts.com/team
Roboto font by Christian Robertson, Google Incchristianrobertson.comwww.google.com/fonts
Inconsolata font by Raph Levien, Google Incwww.levien.comwww.google.com/fonts
hexo: fast, simple & powerful blog frameworkhexo.io
ace editorace.c9.io
virtual-domwww.npmjs.com/package/virtual-dom
Black Granite Water Droplets, William Warbyhttps://www.flickr.com/photos/wwarby