#!/usr/local/bin/perl # $Header: /mhub4/sources/imap-tools/migrateIMAP.pl,v 1.1 2008/10/18 15:09:57 rick Exp $ #************************************************************************* # * # Program name migrateIMAP * # Written by Rick Sanders * # Date 6 May 2008 * # * # Description * # * # This script is used to migrate the e-mail on one IMAP Server * # another. Each users's messages are copied from the "source" * # server to the "destination" server using the IMAP protocol. You * # supply a file with the user's names & passwords. For example: * # * # ./migrateIMAP.pl -S source -D destination -i * # * # Use the -h argument to see the complete list of arguments. * #************************************************************************* &init(); # Get the list of usernames and passwords @users = getUserList( $userlist ); $i=$totalUsers=$children=0; for ($index = 0; $index <= $#users; $index++) { $userinfo = $users[$index]; ($user) = split(/\s*:\s*/, $userinfo); # Start the migration. Unless maxChildren has been set to 1 # fork off child processes to do the migration in parallel. if ($maxChildren == 1) { &migrate ($userinfo); } else { Log("There are $children running") if $debug; if ( $children < $maxChildren ) { Log(" Forking to migrate $user"); if ( $pid = fork ) { # Parent Log (" Parent $$ forked $pid"); } elsif (defined $pid) { # Child Log (" Child process $$ processing $sourceUser"); &migrate($userinfo); Log(" $user is done"); exit 0; } else { Log("Error forking child to migrate $user"); next; } $children++; $children{$pid} = $user; } Log ("I'm PID $$") if $debug; while ( $children >= $maxChildren ) { Log(" $$ - Max children running. Waiting..."); $foundPid = wait; # Wait for a child to terminate if ($? != 0) { Log ("ERROR: PID $foundPid exited with status $?"); } delete $children{$foundPid}; $children--; } Log("OK to launch another user migration") if $debug; } } if ($maxChildren > 1) { Log("All children have been launched, waiting for them to finish"); foreach $pid ( keys(%children) ) { $user = $children{$pid}; Log("Waiting on process $pid ($user) to finish"); waitpid($pid, 0); if ($? != 0) { Log ("ERROR: PID $pid exited with status $?"); } } } &summarize(); $elapsed = sprintf("%.2f", (time()-$start)/3600); Log("Elapsed time $elapsed hours"); Log("Migration completed"); exit; sub migrate { my $user = shift; ($sourceUser,$sourcePwd,$destUser,$destPwd) = split(/\s*:\s*/, $user); Log("Starting migration of $sourceUser"); print STDOUT " Migrating $sourceUser\n"; unless ( $sourcePwd ) { Log("Password not found for $sourceUser, messages will not be migrated"); return; } $conn_timed_out=0; return unless &connectToHost($sourceHost, \$src); return unless &login($sourceUser,$sourcePwd, $src); unless ( &connectToHost( $destHost, \$dst ) ) { &logout( $src ); return; } unless ( &login( $destUser,$destPwd, $dst ) ) { &logout( $src ); return; } namespace( $src, \$srcPrefix, \$srcDelim ); namespace( $dst, \$dstPrefix, \$dstDelim ); $totalUsers++; @mbxs = &getMailboxList($sourceUser, $src); $total = 0; foreach $mbx ( @mbxs ) { $dstmbx = mailboxName( $mbx,$srcPrefix,$srcDelim,$dstPrefix,$dstDelim ); $checkpoint = "$mbx|$sourceHost|$sourceUser|$sourcePwd|"; $checkpoint .= "$destHost|$destUser|$destPwd"; &getMsgList( $mbx, \@msgs, $src ); if ( $#msgs == -1 ) { # Create an empty mailbox &createMbx( $dstmbx, $dst ); $line = pack("A20 A13 A18", $mbx, '', "(0 messages)"); Log(" Copied $line"); next; } $added=0; foreach $_ ( @msgs ) { ($msgnum,$date,$flags) = split(/\|/, $_); alarm $timeout; &fetchMsg( $msgnum, $mbx, \$message, $src ); alarm 0; if ( $conn_timed_out ) { Log("$srcHost timed out"); &reconnect( $checkpoint, $src ); $conn_timed_out = 0; next; } alarm $timeout; &insertMsg( $dst, $dstmbx, *message, $flags, $date ); alarm 0; if ( $conn_timed_out ) { Log("$destHost timed out"); &reconnect( $checkpoint, $dst ); $conn_timed_out = 0; next; } $added++; } $total += $added; $line = pack("A20 A13 A18", $mbx, '', "($added messages)"); Log(" Copied $line"); } # Update the summary file with the totals for this user open(SUM, ">>/tmp/migrateIMAP.sum"); print SUM "$total|$totalBytes\n"; close SUM; $totalBytes = &formatBytes( $totalBytes ); Log(" Copied $total messages $totalBytes"); &logout( $src ); &logout( $dst ); } sub init { use Getopt::Std; use Fcntl; use Socket; use IO::Socket; use sigtrap; use FileHandle; require "ctime.pl"; $version = "1.0.3"; if ( $ENV{OS} =~ /Windows/i ) { print "\nThis version of migrateIMAP does not support Windows since\n"; print "it uses fork() to run multiple simultaneous migration processes\n"; print "for better performance. Please use migrateIMAP-windows.pl\n"; print "instead.\n\n"; exit; } $start = time(); # Set up signal handling $SIG{'ALRM'} = 'signalHandler'; $SIG{'HUP'} = 'signalHandler'; $SIG{'INT'} = 'signalHandler'; $SIG{'TERM'} = 'signalHandler'; $SIG{'URG'} = 'signalHandler'; getopts('S:D:L:i:b:t:n:hIdu'); &usage() if $opt_h; unless ($opt_S and $opt_D ) { &usage(); } $sourceHost = $opt_S; $destHost = $opt_D; $userlist = $opt_i; $logfile = $opt_L; $maxChildren = $opt_n; $usage = $opt_h; $timeout = $opt_t; $unseen = $opt_u; $showIMAP=1 if $opt_I; $debug=1 if $opt_d; $timeout = 45 unless $timeout; $maxChildren = 2 unless $maxChildren; $hostname = `hostname`; $logfile = "migrateIMAP.log" unless $logfile; if ( -e $logfile ) { # Rename the existing logfile $line = `head -n 1 $logfile`; $ts = substr($line,0,16); rename($logfile, "$logfile.$ts"); } open (LOG, ">>$logfile"); select LOG; $| = 1; unlink '/tmp/migrateIMAP.sum' if -e '/tmp/migrateIMAP.sum'; Log("$0 starting"); Log("Renamed old logfile to $logfile.$ts") if $ts; # Validate the arguments and call usage() if necessary $date = &ctime(time); chomp($date); # Determine whether we have SSL support via openSSL and IO::Socket::SSL $ssl_installed = 1; eval 'use IO::Socket::SSL'; if ( $@ ) { $ssl_installed = 0; } } sub usage { print "\nUsage: migrateIMAP.pl -S sourceHost -D destinationHost\n\n"; print "Optional arguments:\n\n"; print " -i \n"; print " -n \n"; print " -L \n"; print " -t \n"; print " -u \n"; print " -d debug mode\n"; print " -I record IMAP protocol exchanges\n\n"; exit; } sub Log { my $line = shift; if ( LOG ) { my @f = localtime( time ); my $timestamp = sprintf( "%02d-%02d-%04d.%02d:%02d:%02d", (1 + $f[ 4 ]), $f[ 3 ], (1900 + $f[ 5 ]), @f[ 2,1,0 ] ); printf LOG "%s %s: %s\n", $timestamp, $$, $line; } } # Make a connection to an IMAP host sub connectToHost { my $host = shift; my $conn = shift; &Log("Connecting to $host") if $debug; ($host,$port) = split(/:/, $host); $port = 143 unless $port; # We know whether to use SSL for ports 143 and 993. For any # other ones we'll have to figure it out. $mode = sslmode( $host, $port ); if ( $mode eq 'SSL' ) { unless( $ssl_installed == 1 ) { warn("You must have openSSL and IO::Socket::SSL installed to use an SSL connection"); Log("You must have openSSL and IO::Socket::SSL installed to use an SSL connection"); exit; } Log("Attempting an SSL connection") if $debug; $$conn = IO::Socket::SSL->new( Proto => "tcp", SSL_verify_mode => 0x00, PeerAddr => $host, PeerPort => $port, ); unless ( $$conn ) { $error = IO::Socket::SSL::errstr(); Log("Error connecting to $host: $error"); warn("Error connecting to $host: $error"); exit; } } else { # Non-SSL connection Log("Attempting a non-SSL connection") if $debug; $$conn = IO::Socket::INET->new( Proto => "tcp", PeerAddr => $host, PeerPort => $port, ); unless ( $$conn ) { Log("Error connecting to $host:$port: $@"); warn "Error connecting to $host:$port: $@"; exit; } } Log("Connected to $host on port $port"); } sub sslmode { my $host = shift; my $port = shift; my $mode; # Determine whether to make an SSL connection # to the host. Return 'SSL' if so. if ( $port == 143 ) { # Standard non-SSL port return ''; } elsif ( $port == 993 ) { # Standard SSL port return 'SSL'; } unless ( $ssl_installed ) { # We don't have SSL installed on this machine return ''; } # For any other port we need to determine whether it supports SSL my $conn = IO::Socket::SSL->new( Proto => "tcp", SSL_verify_mode => 0x00, PeerAddr => $host, PeerPort => $port, ); if ( $conn ) { close( $conn ); $mode = 'SSL'; } else { $mode = ''; } return $mode; } # login # # login in at the source host with the user's name and password # sub login { my $user = shift; my $pwd = shift; my $conn = shift; &sendCommand ($conn, "1 LOGIN $user $pwd"); while (1) { &readResponse ( $conn ); if ($response =~ /^1 OK/i) { last; } elsif ($response =~ /^1 NO|^1 BAD/) { Log ("$user login failed: unexpected LOGIN response: $response"); return 0; } } &Log("Logged in as $user") if $debug; return 1; } # getMailboxList # # get a list of the user's mailboxes from the source host # sub getMailboxList { my $user = shift; my $conn = shift; my @mbxs; my @mailboxes; # Get a list of the user's mailboxes # if ( $mbxList ) { # The user has supplied a list of mailboxes so only processes # the ones in that list @mbxs = split(/,/, $mbxList); foreach $mbx ( @mbxs ) { trim( *mbx ); push( @mailboxes, $mbx ); } return @mailboxes; } if ($debug) { Log("Get list of user's mailboxes",2); } sendCommand ($conn, "1 LIST \"\" *"); undef @response; while ( 1 ) { readResponse ($conn); if ( $response =~ /^1 OK/i ) { last; } elsif ( $response !~ /^\*/ ) { Log ("unexpected response: $response"); return 0; } } undef @mbxs; for $i (0 .. $#response) { $response[$i] =~ s/\s+/ /; if ( $response[$i] =~ /"$/ ) { $response[$i] =~ /\* LIST \((.*)\) "(.+)" "(.+)"/i; $mbx = $3; } else { $response[$i] =~ /\* LIST \((.*)\) "(.+)" (.+)/i; $mbx = $3; } $mbx =~ s/^\s+//; $mbx =~ s/\s+$//; if ($response[$i] =~ /NOSELECT/i) { if ($debug) { Log("$mbx is set NOSELECT,skip it",2); } next; } if (($mbx =~ /^\#/) && ($user ne 'anonymous')) { # Skip public mbxs unless we are migrating them next; } if ($mbx =~ /^\./) { # Skip mailboxes starting with a dot next; } push ( @mbxs, $mbx ) if $mbx ne ''; } if ( $mbxList ) { # The user has supplied a list of mailboxes so only processes # those @mbxs = split(/,/, $mbxList); } return @mbxs; } # getMsgList # # Get a list of the user's messages in the indicated mailbox on # the source host # sub getMsgList { my $mailbox = shift; my $msgs = shift; my $conn = shift; my $seen; my $empty; my $msgnum; my $from; my $flags; &trim( *mailbox ); &sendCommand ($conn, "1 EXAMINE \"$mailbox\""); undef @response; $empty=0; while ( 1 ) { &readResponse ( $conn ); if ( $response =~ / 0 EXISTS/i ) { $empty=1; } if ( $response =~ /^1 OK/i ) { # print STDERR "response $response\n"; last; } elsif ( $response !~ /^\*/ ) { &Log ("unexpected response: $response"); # print STDERR "Error: $response\n"; return 0; } } &Log("Fetch the header info") if $debug; &sendCommand ( $conn, "1 FETCH 1:* (uid flags internaldate body[header.fields (From Date)])"); undef @response; while ( 1 ) { &readResponse ( $conn ); return if $conn_timed_out; if ( $response =~ /^1 OK/i ) { last; } elsif ( $response =~ /could not be processed/i ) { Log("Error: response from server: $response"); last; } elsif ( $response =~ /^1 NO\s/i ) { last; } } @msgs = (); $flags = ''; for $i (0 .. $#response) { $seen=0; $_ = $response[$i]; last if /OK FETCH complete/; if ( $response[$i] =~ /^From: (.+)/ ) { $from = $1; } if ($response[$i] =~ /FLAGS/) { # Get the list of flags $response[$i] =~ /FLAGS \(([^\)]*)/; $flags = $1; $flags =~ s/\\Recent//; } if ( $response[$i] =~ /INTERNALDATE/) { $response[$i] =~ /INTERNALDATE (.+) BODY/; # $response[$i] =~ /INTERNALDATE "(.+)" BODY/; $date = $1; $date =~ /"(.+)"/; $date = $1; $date =~ s/"//g; } if ( $response[$i] =~ /\* (.+) FETCH/ ) { ($msgnum) = split(/\s+/, $1); } if ( $msgnum && $from && $date ) { if ( $unseen ) { push (@$msgs,"$msgnum|$date|$flags") unless $flags =~ /Seen/i; } else { push (@$msgs,"$msgnum|$date|$flags"); } $msgnum = $from = $date = ''; } } } sub fetchMsg { my $msgnum = shift; my $mbx = shift; my $message = shift; my $conn = shift; &Log(" Fetching msg $msgnum...") if $debug; $$mesage = ''; &sendCommand( $conn, "1 FETCH $msgnum (rfc822)"); @a = (); while (1) { &readResponse ($conn); &Log ("Unable to fetch message - connection timeout") if ($conn_timed_out); push (@a, $response); if ( $response =~ /^1 OK/i ) { last; } elsif ($response =~ /message number out of range/i) { &Log ("Error fetching uid $uid: out of range",2); $stat=0; last; } elsif ($response =~ /Bogus sequence in FETCH/i) { &Log ("Error fetching uid $uid: Bogus sequence in FETCH",2); $stat=0; last; } elsif ( $response =~ /message could not be processed/i ) { &Log("Message could not be processed, skipping it"); push(@errors,"Message could not be processed, skipping it"); $stat=0; last; } elsif ($response =~ /^\*\s+$msgnum\s+FETCH\s+\(.*RFC822\s+\{[0-9]+\}/i) { ($len) = ($response =~ /^\*\s+$msgnum\s+FETCH\s+\(.*RFC822\s+\{([0-9]+)\}/i); $cc = 0; $$message = ""; while ( $cc < $len ) { # &Log ("Already read $cc bytes of $len - waiting on " . ($len - $cc)) if $debug; $n = 0; $n = read ($conn, $segment, $len - $cc); # $n = read ($conn, $segment, ($len - $cc > 4096 ? 4096 : $len-$cc)); # &Log ("Read $n bytes") if $debug; if ( $n == 0 ) { &Log ("unable to read $len bytes"); return 0; } $$message .= $segment; $cc += $n; } } } } # # readResponse # # This subroutine reads and formats an IMAP protocol response from an # IMAP server on a specified connection. # sub readResponse { my $fd = shift; exit unless defined $fd; $response = <$fd>; chop $response; $response =~ s/\r//g; push (@response,$response); Log ("<< *** Connection timeout ***") if $conn_timed_out; Log ("<< $response") if $showIMAP; } # sendCommand # # This subroutine formats and sends an IMAP protocol command to an # IMAP server on a specified connection. # sub sendCommand { local($fd) = shift @_; local($cmd) = shift @_; print $fd "$cmd\r\n"; Log (">> $cmd") if $showIMAP; } # # log out from the host # sub logout { my $conn = shift; undef @response; &sendCommand ($conn, "1 LOGOUT"); while ( 1 ) { &readResponse ($conn); next if $response =~ /APPEND complete/i; # Ignore strays if ( $response =~ /^1 OK/i ) { last; } elsif ( $response !~ /^\*/ ) { Log("unexpected logout response $response"); last; } } close $conn; return; } # trim # # remove leading and trailing spaces from a string sub trim { local (*string) = @_; $string =~ s/^\s+//; $string =~ s/\s+$//; return; } # insertMsg # # This routine inserts an RFC822 messages into a user's folder # sub insertMsg { local ($conn, $mbx, *message, $flags, $date) = @_; local ($lsn,$lenx); &Log(" Inserting message") if $debug; $lenx = length($message); $totalBytes = $totalBytes + $lenx; $totalMsgs++; # Create the mailbox unless we have already done so if ($destMbxs{"$mbx"} eq '') { &createMbx( $mbx, $dst ); } $destMbxs{"$mbx"} = '1'; $flags =~ s/\\Recent//i; &sendCommand ($conn, "1 APPEND \"$mbx\" ($flags) \"$date\" \{$lenx\}"); &readResponse ($conn); if ($conn_timed_out) { &Log ("unexpected response timeout appending message"); push(@errors,"Error appending message to $mbx for $user"); return 0; } if ( $response !~ /^\+/ ) { &Log ("unexpected APPEND response: >$response<"); # next; push(@errors,"Error appending message to $mbx for $user"); return 0; } print $conn "$message\r\n"; undef @response; while ( 1 ) { &readResponse ($conn); if ( $response =~ /^1 OK/i ) { last; } elsif ( $response !~ /^\*/ ) { &Log ("Unexpected APPEND response: >$response<"); # next; return 0; } } return; } sub createMbx { my $mbx = shift; my $conn = shift; # Create a mailbox &sendCommand ($conn, "1 CREATE \"$mbx\""); while ( 1 ) { &readResponse ($conn); last if $response =~ /^1 OK|already exists /i; if ( $response !~ /^\*/ ) { if (!($response =~ /already exists|reserved mailbox name/i)) { # &Log ("WARNING: $response"); } last; } } } sub formatBytes { my $bytes = shift; # Format the number nicely if ( length($bytes) >= 10 ) { $bytes = $bytes/1000000000; $tag = 'GB'; } elsif ( length($bytes) >= 7 ) { $bytes = $bytes/1000000; $tag = 'MB'; } else { $bytes = $bytes/1000; $tag = 'KB'; } # commafy $_ = $bytes; 1 while s/^([-+]?\d+)(\d{3})/$1,$2/; $bytes = sprintf("%.2f", $_) . " $tag"; return $bytes; } sub getUserList { my $fn = shift; @users = (); unless ( -e $fn ) { Log("Fatal error reading $fn: $!"); exit; } open(L, "<$fn") or die $!; while ( ) { chomp; s/^\s+//; next if /#/; push( @users, $_ ); } close L; return @users; } sub selectMbx { my $mbx = shift; my $conn = shift; Log("selecting mbx $mbx on $conn"); &sendCommand ($conn, "1 EXAMINE \"$mbx\""); undef @response; $empty=0; while ( 1 ) { &readResponse ( $conn ); if ( $response =~ /^1 OK/i ) { # print STDERR "response $response\n"; last; } elsif ( $response !~ /^\*/ ) { &Log ("unexpected response: $response"); return 0; } } } # Reconnect to a server after a timeout error. # sub reconnect { my $checkpoint = shift; my $conn = shift; Log("This is reconnect, conn is $conn") if $debug; &logout( $conn ); close $conn; sleep 5; ($mbx,$shost,$suser,$spwd,$dhost,$duser,$dpwd) = split(/\|/, $checkpoint); if ( $conn eq $src ) { $host = $shost; $user = $suser; $pwd = $spwd; } else { $host = $dhost; $user = $duser; $pwd = $dpwd; } &connectToHost($host,$conn); &login($user,$pwd,$conn); &selectMbx( $mbx, $conn ); &createMbx( $mbx, $dst ); # Just in case Log("leaving reconnect"); } # Handle signals sub signalHandler { my $sig = shift; if ( $sig eq 'ALRM' ) { Log("Caught a SIG$sig signal, timeout error"); $conn_timed_out = 1; } else { Log("Caught a SIG$sig signal, shutting down"); exit; } } # Get the total message count and bytes and write # it to the log. sub summarize { # Each child appends its totals to /tmp/migrateEmail.sum so # we read the lines and add up the grand totals. $totalUsers=$totalMsgs=$totalBytes=0; open(SUM, " ) { chomp; ($msgs,$bytes) = split(/\|/, $_); $totalUsers++; $totalMsgs += $msgs; $totalBytes += $bytes; } $_ = $totalMsgs; 1 while s/^([-+]?\d+)(\d{3})/$1,$2/; # Commafy the message total $totalMsgs = $_; $totalBytes = &formatBytes( $totalBytes ); Log("Summary of migration"); Log("Migrated $totalUsers users, $totalMsgs messages, $totalBytes."); } sub namespace { my $conn = shift; my $prefix = shift; my $delimiter = shift; # Query the server with NAMESPACE so we can determine its # mailbox prefix (if any) and hierachy delimiter. @response = (); sendCommand( $conn, "1 NAMESPACE"); while ( 1 ) { readResponse( $conn ); if ( $response =~ /^1 OK/i ) { last; } elsif ( $response =~ /NO|BAD/i ) { Log("Unexpected response to NAMESPACE command: $response"); last; } } foreach $_ ( @response ) { if ( /NAMESPACE/i ) { my $i = index( $_, '((' ); my $j = index( $_, '))' ); my $val = substr($_,$i+2,$j-$i-3); ($$prefix,$$delimiter) = split( / /, $val ); $$prefix =~ s/"//g; $$delimiter =~ s/"//g; last; } last if /^NO|^BAD/; } if ( $debug ) { Log("prefix $$prefix"); Log("delim $$delimiter"); } } sub mailboxName { my $srcmbx = shift; my $srcPrefix = shift; my $srcDelim = shift; my $dstPrefix = shift; my $dstDelim = shift; my $dstmbx; # Adjust the mailbox name if the source and destination server # have different mailbox prefixes or hierarchy delimiters. if ( $debug ) { Log("src mbx $srcmbx"); Log("src prefix $srcPrefix"); Log("src delim $srcDelim"); Log("dst prefix $dstPrefix"); Log("dst delim $dstDelim"); } if ( ($srcPrefix eq $dstPrefix) and ($srcDelim eq $dstDelim) ) { # No adjustments necessary $dstmbx = $srcmbx; if ( $root_mbx ) { # Put folders under a 'root' folder on the dst $dstmbx =~ s/^$dstPrefix//; $dstDelim =~ s/\./\\./g; $dstmbx =~ s/^$dstDelim//; $dstmbx = $dstPrefix . $root_mbx . $dstDelim . $dstmbx; if ( uc($srcmbx) eq 'INBOX' ) { # Special case for the INBOX $dstmbx =~ s/INBOX$//i; $dstmbx =~ s/$dstDelim$//; } $dstmbx =~ s/\\//g; } return $dstmbx; } $srcmbx =~ s#^$srcPrefix##; $dstmbx = $srcmbx; if ( $srcDelim ne $dstDelim ) { # Need to substitute the dst's hierarchy delimiter for the src's one $srcDelim = '\\' . $srcDelim if $srcDelim eq '.'; $dstDelim = "\\" . $dstDelim if $dstDelim eq '.'; $dstmbx =~ s#$srcDelim#$dstDelim#g; $dstmbx =~ s/\\//g; } if ( $srcPrefix ne $dstPrefix ) { # Replace the source prefix with the dest prefix $dstmbx =~ s#^$srcPrefix## if $srcPrefix; if ( $dstPrefix ) { $dstmbx = "$dstPrefix$dstmbx" unless uc($srcmbx) eq 'INBOX'; } $dstDelim = "\\$dstDelim" if $dstDelim eq '.'; $dstmbx =~ s#^$dstDelim##; } if ( $root_mbx ) { # Put folders under a 'root' folder on the dst $dstDelim =~ s/\./\\./g; $dstmbx =~ s/^$dstPrefix//; $dstmbx =~ s/^$dstDelim//; $dstmbx = $dstPrefix . $root_mbx . $dstDelim . $dstmbx; if ( uc($srcmbx) eq 'INBOX' ) { # Special case for the INBOX $dstmbx =~ s/INBOX$//i; $dstmbx =~ s/$dstDelim$//; } $dstmbx =~ s/\\//g; } return $dstmbx; }