Install Xen 4.1.4 with Remus and DRBD on Ubuntu 12.10

This example uses two servers, "left" (server #1), and "right" (server #2).
=Install a new Ubuntu 12.10 system=
Both Servers
Choose 'Use entire disk with LVM', but only use like 15GB for the system to leave the rest for VMs
Network setup
apt-get install bridge-utils
vi /etc/network/interfaces
#
auto xenbr0
iface xenbr0 inet static
address 10.0.1.50
netmask 255.255.255.0
gateway 10.0.1.1
dns-nameservers 10.0.1.1
bridge_ports eth0
auto eth0
iface eth0 inet manual
#
/etc/init.d/networking restart
vi /etc/hosts
#
10.0.1.50 left
10.0.1.52 right
#
=Install Xen=
==Install Xen Pre-requisites==
Server #1
Short-cut install for dependencies
apt-get build-dep xen
apt-get install libc6-dev libglib2.0-dev libyajl-dev yajl-tools libbz2-dev bison flex zlib1g-dev git-core texinfo debhelper debconf-utils debootstrap fakeroot
If you're missing something, here's the full list
apt-get install libjpeg-turbo8-dev
apt-get install libvncserver-dev
apt-get install git-core mercurial screen tcpdump minicom ntp ntpdate tree debootstrap bcc bin86 gawk bridge-utils
apt-get install iproute libcurl3 libcurl4-openssl-dev bzip2 module-init-tools transfig tgif texinfo pciutils-dev build-essential make gcc 
apt-get install libc6-dev zlib1g-dev python python-dev python-twisted libncurses5-dev patch libvncserver-dev iasl libbz2-dev 
apt-get install e2fslibs-dev uuid-dev libtext-template-perl autoconf debhelper debconf-utils docbook-xml docbook-xsl dpatch xsltproc rcconf bison flex 
apt-get install gcc-multilib ocaml-findlib libyajl-dev yajl-tools libglib2.0-dev libsdl-ttf2.0-0 libsdl-ttf2.0-dev
==Install Xen 4.1.4==
Server #1
apt-get install transfig
http://bits.xensource.com/oss-xen/release/4.1.4/xen-4.1.4.tar.gz
tar zxvf xen-4.1.4.tar.gz
cd xen-4.1.4
make world
mv ./dist/install/usr/lib64/* ./dist/install/usr/lib/
mv ./dist/install/usr/lib64/xen/bin/* ./dist/install/usr/lib/xen/bin/
tar zcvf xen-dist.tgz ./xen-4.1.4/dist
scp ./xen-dist.tgz user@10.0.1.52:.
Server #2
tar zxvf xen-dist.tgz
Both servers
cd /home/user/xen-4.1.4/dist
./install.sh
update-grub
ls -al /boot/xen*
-rw-r--r-- 1 root root 737577 Feb 18 10:53 /boot/xen-4.1.4.gz
lrwxrwxrwx 1 root root 12 Feb 18 10:53 /boot/xen-4.1.gz -> xen-4.1.4.gz
lrwxrwxrwx 1 root root 12 Feb 18 10:53 /boot/xen-4.gz -> xen-4.1.4.gz
lrwxrwxrwx 1 root root 12 Feb 18 10:53 /boot/xen.gz -> xen-4.1.4.gz
-rw-r--r-- 1 root root 14003510 Feb 18 10:53 /boot/xen-syms-4.1.4
update-rc.d xencommons defaults 19 18
update-rc.d xend defaults 20 21
update-rc.d xendomains defaults 21 20
update-rc.d xen-watchdog defaults 22 23
grep Xen /boot/grub/grub.cfg
# You should see "Ubuntu GNU\/Linux, with Xen hypervisor" in there, that's where I got the below from
sed -i 's/GRUB_DEFAULT=.*\+/GRUB_DEFAULT="Ubuntu GNU\/Linux, with Xen hypervisor"/' /etc/default/grub
update-grub
reboot
==Optional: Test Xen==
Either server
pvs
lvcreate -L 8G -n ubuntu right
mkdir -p /var/lib/xen/images/ubuntu-netboot
cd /var/lib/xen/images/ubuntu-netboot
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/initrd.gz
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/vmlinuz
vi /etc/xen/ubuntu.cfg
#
name = "ubuntu"
memory = 256
disk = ['phy:/dev/left/ubuntu,xvda,w']
vif = [' ']
kernel = "/var/lib/xen/images/ubuntu-netboot/vmlinuz"
ramdisk = "/var/lib/xen/images/ubuntu-netboot/initrd.gz"
extra = "debian-installer/exit/always_halt=true -- console=hvc0"
#
xm create /etc/xen/ubuntu.cfg -c
# run the install
vi /etc/xen/ubuntu.cfg
# 
bootloader = "pygrub"
# comment out 'kernel' 'ramdisk' and 'extra'
#
# Press Ctrl+] to exit the console view
xm create /etc/xen/ubuntu.cfg -c
# see it running
xm list
Hint: Tapping escape while booting should display grub menu if needed for debugging
=Install DRBD=
Note: Installs /etc/xen/scripts/block-drbd for xen drbd block support as well
Server #1
apt-get install autoconf build-essential
wget http://remusha.wikidot.com/local--files/configuring-and-installing-remus/drbd-8.3.11-remus.tar.gz
cd ./drbd-8.3.11
./autogen.sh
./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc --with-km
make
make install
cd drbd
make clean all
make install
vi /etc/modules
# add drbd
#
tar zcvf drbd.tgz ./drbd-8.3.11
scp ./drbd.tgz user@10.0.1.52:.
Server #2
# on 1.50
tar zxvf ./drbd.tgz
cd ./drbd-8.3.11
make install
cd drbd
make install
vi /etc/modules
# add drbd
#
Both Servers
cp /home/user/drbd-8.3.11/scripts/global_common.conf.protoD /etc/drbd.d/global_common.conf
cp /home/user/drbd-8.3.11/scripts/testvms_protoD.res /etc/drbd.d/SystemHA_protoD.res
uname -a # center
lvcreate -n test -L 10G center
vi /etc/drbd.d/SystemHA_protoD.res
#
resource drbd-vm {
 device /dev/drbd1;
 disk /dev/center/test;
 meta-disk internal;
 on left {
 address 10.0.1.50:7791;
 }
 on center {
 address 10.0.1.51:7791;
 }
}
#
#Create the meta-data for the SystemHA-disk and then bring up the resource. Do this on both machines
drbdadm create-md drbd-vm
###answer y or yes for all questions, in the above command
drbdadm up drbd-vm
###Sanity check. You should see something like this in the output
root@null:~$ cat /proc/drbd 
version: 8.3.9 (api:88/proto:86-95)
GIT-hash: 3ba4fc581d6215744597b3d4c525db276ce000ee build by root@null, 2011-05-14 19:33:56
 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent D r-----
Server #1 - this will override all of the DRBD data on server #2
drbdadm -- --overwrite-data-of-peer primary drbd-vm
Watch the progress of the transfer (either server)
cat /proc/drbd
Note:When configuring your VM, use drbd:drbd-vm for the disk. not phy:/… or any other format
Note:It's one VM per DRBD disk like this, so you have to create a new one for each VM
Note:You should do some research on DRBD. Do not put both disks in the primary mode "manually". It will be automatically done by the block-drbd script when Remus is running. When Remus is not enabled for a VM, only the local backing disk will be in primary mode. The remote disk will be in secondary mode.
===Setup Remus===
Both servers
vi /etc/xen/xend-config.sxp
# Ensure the following are uncommented
(xend-relocation-server yes)
(xend-relocation-port 8002)
(xend-relocation-address )
(xend-relocation-hosts-allow )
#
vi /etc/modules
#
sch_plug
sch_prio
sch_ingress
cls_basic
cls_tcindex
cls_u32
act_mirred
ifb
#
==Test VM with Remus==
Server #1
mkdir -p /var/lib/xen/images/ubuntu-netboot
cd /var/lib/xen/images/ubuntu-netboot
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/initrd.gz
wget http://ubuntu.c3sl.ufpr.br/ubuntu/dists/precise/main/installer-amd64/current/images/netboot/xen/vmlinuz
vi /etc/xen/rt.cfg
#
name = "rt"
memory = 256
disk = [ 'drbd:drbd-vm,xvda,w' ]
vif = [' ']
kernel = "/var/lib/xen/images/ubuntu-netboot/vmlinuz"
ramdisk = "/var/lib/xen/images/ubuntu-netboot/initrd.gz"
extra = "debian-installer/exit/always_halt=true -- console=hvc0"
#
xm create /etc/xen/rt.cfg -c
# run the install
sudo ln -s /usr/lib/xen-4.1/bin/pygrub /usr/bin/pygrub
vi /etc/xen/rt.cfg
# 
bootloader = "pygrub"
# comment out 'kernel' 'ramdisk' and 'extra'
#
# Press Ctrl+] to exit the console view
xm create /etc/xen/rt.cfg -c
# see it running
xm list
# test live migration, if you can't live migrate, then Remus isn't going to work either
xm migrate rt right
# try remus ("rt" is the domU, and "right" is the other server, assuming we're running from "left" here)
remus -i 40 rt right
# You should get a whole bunch of these messages:
#
PROF: flushed memory at 1361239602.516625
PROF: suspending at 1361239602.554033
#
Now if you kill the server in some way, the VM will come up automatically on the secondary.
Category:Xen Category:Users Category:HowTo Category:Ubuntu Category:High_Availability