Installing NixOS (Encrypted BTRFS Impermanance)
Intro
This details the steps to create a LUKS encrypted BTRFS NixOS installation with impermanance, only keeping the settings that are required in between reboots.
Features:
- UEFI Boot w/GRUB
- Encrypted Btrfs File System with hourly snapshots on
/homeand/var/local - Swap File instead of Swap Partition w/Hibernate Support
This is a work in progress using the standard configuration.nix setup.
Todo
- Setup FDE instead of Partition 2 Errors
Disk Setup
Setup Variables
DISK_EFI_SIZE_MB=512
DISK=/dev/vda
PASSWORD=password
Wipe and Partition Disk (Swap File)
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
--new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
--new=2:0:0 --typecode=2:8200 --change-name=2:pool0_0 \
"${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2
Create Encrypted Disk and Initial Filesystems
echo "${PASSWORD}" | cryptsetup --verify-passphrase -v luksFormat "${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"
Create BTRFS Subvolumes
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
Take a readonly snapshot of the root subvolume, which will get rolled back to on every boot.
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
Mount Subvolumes and Partitions
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots
mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log
mkdir -p /mnt/boot/efi
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot/efi
Generate NixOS Hardware Configuration
nixos-generate-config --root /mnt
File Based Swap
sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/& \
swapDevices = [{\
device = \"\/swap\/swapfile\";\
size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n }];\n/' /mnt/etc/nixos/hardware-configuration.nix
sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix
truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile
sed -i "/boot.extraModulePackages/a\\
boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;" /mnt/etc/nixos/hardware-configuration.nix
Force var_log and persist subvolume to be available for boot
sed -i "/subvol=var_log/a\ neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix
sed -i "/subvol=persist/a\ neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix
Force BTRFS mount options
Nixos-generate config isn't smart enough to add options as per here.
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\" \];|g" /mnt/etc/nixos/hardware-configuration.nix
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix
Allow discards to encrypted root
sed -i '/boot.initrd.luks.devices/a\ boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix
Configure Install
Choose either the configuration.nix track or Nix Flakes
configuration.nix
/etc/nixos/configuration.nix
{ config, pkgs, ... }:
{
imports =
[
./hardware-configuration.nix
];
boot = {
loader = {
efi = {
canTouchEfiVariables = false;
};
grub = {
enable = true;
device = "nodev"; # No device for EFI
efiSupport = true;
enableCryptodisk = false;
useOSProber = false;
efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
};
};
supportedFilesystems = [
"btrfs"
"fat" "vfat" "exfat" "ntfs" # Microsoft
"cifs" # Windows Network Share
];
};
networking = {
hostName = "beef";
networkmanager.enable = true;
};
i18n.defaultLocale = "en_US.UTF-8";
security.sudo.extraConfig = ''
Defaults lecture = never
'';
services = {
cinnamon.apps.enable = false ;
btrbk = {
instances."btrbak" = {
onCalendar = "*-*-* *:00:00";
settings = {
timestamp_format = "long";
snapshot_preserve_min = "2d";
preserve_day_of_week = "sunday" ;
preserve_hour_of_day = "0" ;
target_preserve = "48h 10d 4w 12m 10y" ;
volume."/home" = {
snapshot_create = "always";
subvolume = ".";
snapshot_dir = ".snapshots";
};
volume."/var/local" = {
snapshot_create = "always";
subvolume = ".";
snapshot_dir = ".snapshots";
};
};
openssh = {
enable = true;
settings = {
PermitRootLogin = "yes" ;
};
};
xserver = {
enable = true;
layout = "us";
libinput.enable = true;
displayManager.gdm.enable = true ;
desktopManager.cinnamon.enable = true;
};
};
system.stateVersion = "23.05";
systemd.services = {
create-swapfile = {
serviceConfig.Type = "oneshot";
wantedBy = [ "swap-swapfile.swap" ];
script = ''
swapfile="/swap/swapfile"
if [ -f "$swapfile" ]; then
echo "Swap file $swapfile already exists, taking no action"
else
echo "Setting up swap file $swapfile"
${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
fi
'';
};
};
time.timeZone = "America/Vancouver";
users.users.dave = {
description = "Dave Conroy";
createHome = true ;
home = "/home/dave" ;
shell = "/bin/sh" ; # This is actually bash
group = "users" ;
extraGroups = [ "wheel" "docker" ];
uid = 2323;
isNormalUser = true;
hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
packages = with pkgs; [
];
};
}
Flakes
Install NixOS
nixos-install
Configuring Impermanence
Boot into new system make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on maping files that need to be available upon each reboot.
Create differences script
#!/usr/bin/env bash
_tmp_root=$(mktemp -d)
mkdir -p "${_tmp_root}"
mount -o subvol=/ /dev/mapper/pool0_0 "${_tmp_root}" > /dev/null 2>&1
set -euo pipefail
OLD_TRANSID=$(sudo btrfs subvolume find-new ${_tmp_root}/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }
sudo btrfs subvolume find-new "${_tmp_root}/root" "$OLD_TRANSID" | sed '$d' | cut -f17- -d' ' | sort | uniq |
while read path; do
path="/$path"
if [ -L "$path" ]; then
: # The path is a symbolic link, so is probably handled by NixOS already
elif [ -d "$path" ]; then
: # The path is a directory, ignore
else
echo "$path"
fi
done
umount "${_tmp_root}"
rm -rf "${_tmp_root}"
Create Configuration to reset root on each reboot
boot.initrd.postDeviceCommands = pkgs.lib.mkBefore ''
mkdir -p /mnt
mount -o subvol=/ /dev/mapper/pool0_0 /mnt
btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
while read subvolume; do
echo "Deleting /$subvolume subvolume"
btrfs subvolume delete "/mnt/$subvolume"
done &&
echo "Deleting /root subvolume" &&
btrfs subvolume delete /mnt/root
echo "Restoring blank /root subvolume"
btrfs subvolume snapshot /mnt/root-blank /mnt/root
umount /mnt
'';
Configuring Impermanence
{ config, pkgs, ... }:
let
impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in
{
imports = [ "${impermanence}/nixos.nix" ];
# this folder is where the files will be stored (don't put it in tmpfs)
environment.persistence."/persist" = {
hideMounts = true ;
directories = [
"/etc/nixos" # NixOS
"/etc/NetworkManager" # NetworkManager
{ directory = "/var/lib/colord";
user = "colord";
group = "colord";
mode = "u=rwx,g=rx,o="; } # Colord Profiles
"/var/lib/bluetooth" # Bluetooth
"/var/lib/docker" # Docker
];
files = [
"/etc/ssh/ssh_host_*" # SSH
"/etc/machine-id" # Needed for SystemD Journal
"/var/lib/NetworkManager/secret_key" # Network Manager
"/var/lib/NetworkManager/seen-bssids" # Network Manager
"/var/lib/NetworkManager/timestamps" # Network Manager
{ file = "/etc/nix/id_rsa"; parentDirectory = { mode = "u=rwx,g=,o="; }; } # Nix
];
};cnn.com
Persistence Rules
Bluetooth
systemd.tmpfiles.rules = [
"L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
];
##
sudo mkdir -p /persist/var/lib/bluetooth
Docker
virtualisation = {
docker.enable = true;
};
systemd.tmpfiles.rules = [
"L /var/lib/docker - - - - /persist/var/lib/docker"
];
##
sudo systemctl stop docker
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker
Network Manager
environment.etc."NetworkManager/system-connections" = {
source = "/persist/etc/NetworkManager/system-connections/";
};
systemd.tmpfiles.rules = [
"L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
"L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
"L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
];
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/
NixOS
environment.etc = {
nixos.source = "/persist/etc/nixos";
NIXOS.source = "/persist/etc/NIXOS";
machine-id.source = "/persist/etc/machine-id";
};
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id
OpenSSH
services.openssh = {
enable = true;
hostKeys = [
{
path = "/persist/etc/ssh/ssh_host_ed25519_key";
type = "ed25519";
}
{
path = "/persist/etc/ssh/ssh_host_rsa_key";
type = "rsa";
bits = 4096;
}
];
};
##
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
Scratchpad
Unmount commands
umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt
Restart Install over and over
umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/swap
umount /mnt/var/local
umount /mnt/var/log
umount /mnt
PASSWORD=password
DISK_EFI_SIZE_MB=512
DISK=/dev/vda
wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
--new=1:0:+"${DISK_EFI_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
--new=2:0:0 --typecode=2:8300 --change-name=2:pool0_0 \
"${DISK}"
EFI_PARTITION=1
CRYPT_PARTITION=2
echo "${PASSWORD}" | cryptsetup luksFormat "${DISK}""${CRYPT_PARTITION}"
echo "${PASSWORD}" | cryptsetup open "${DISK}""${CRYPT_PARTITION}" pool0_0
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}""${EFI_PARTITION}"
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
btrfs subvolume create /mnt/swap
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/home/.snapshots
mount -o subvol=home/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home/.snapshots
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/swap
mount -o subvol=swap,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/swap
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/local/.snapshots
mount -o subvol=var_local/snapshots,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local/.snapshots
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot
nixos-generate-config --root /mnt
sed -i '1h;1!H;$!d;x;s/.*fileSystems.*};\n[^\n]*/& \
swapDevices = [{\
device = \"\/swap\/swapfile\";\
size = (1024 * 16) + (1024 * 2); # RAM size + 2 GB\n }];\n/' /mnt/etc/nixos/hardware-configuration.nix
sed -i '/swapDevices = \[ \];/d' /mnt/etc/nixos/hardware-configuration.nix
truncate -s 0 /mnt/swap/swapfile
chattr +C /mnt/swap/swapfile
sed -i "/boot.extraModulePackages/a\\
boot.kernelParams = [ \"resume_offset=$(btrfs inspect-internal map-swapfile -r /mnt/swap/swapfile)\" ];\n\
boot.resumeDevice = \"/dev/disk/by-uuid/$(blkid | grep '/dev/mapper/pool0_0' | awk '{print $2}' | cut -d '"' -f 2)\" ;" /mnt/etc/nixos/hardware-configuration.nix
sed -i "/subvol=var_log/a\ neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix
sed -i "/subvol=persist/a\ neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\" \];|g" /mnt/etc/nixos/hardware-configuration.nix
sed -i 's|"subvol=swap" "compress=zstd"|"subvol=swap" "compress=none"|g' /mnt/etc/nixos/hardware-configuration.nix
sed -i '/boot.initrd.luks.devices/a\ boot.initrd.luks.devices."pool0_0".allowDiscards = true;' /mnt/etc/nixos/hardware-configuration.nix
rm -rf /mnt/etc/nixos/configuration.nix
nano /mnt/etc/nixos/configuration.nix
Full configuration.nix
{ config, pkgs, ... }:
let
impermanence = builtins.fetchTarball "https://github.com/nix-community/impermanence/archive/master.tar.gz";
in
{
imports =
[
./hardware-configuration.nix
"${impermanence}/nixos.nix"
];
boot = {
initrd.postDeviceCommands = pkgs.lib.mkBefore ''
mkdir -p /mnt
mount -o subvol=/ /dev/mapper/pool0_0 /mnt
btrfs subvolume list -o /mnt/root | cut -f9 -d' ' |
while read subvolume; do
echo "Deleting /$subvolume subvolume"
btrfs subvolume delete "/mnt/$subvolume"
done &&
echo "Deleting /root subvolume" &&
btrfs subvolume delete /mnt/root
echo "Restoring blank /root subvolume"
btrfs subvolume snapshot /mnt/root-blank /mnt/root
umount /mnt
'';
loader = {
efi = {
canTouchEfiVariables = false;
};
grub = {
enable = true;
device = "nodev"; # No device for EFI
efiSupport = true;
enableCryptodisk = false;
useOSProber = false;
efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
};
};
kernel.sysctl = {
"vm.dirty_ratio" = 6; # sync disk when buffer reach 6% of memory
};
kernelPackages = pkgs.linuxPackages_latest; # Latest kernel
supportedFilesystems = [
"btrfs"
"fat" "vfat" "exfat" "ntfs" # Microsoft
"cifs" # Windows Network Share
];
};
networking = {
hostName = "beef";
networkmanager.enable = true;
};
i18n.defaultLocale = "en_US.UTF-8";
nix = {
gc = {
automatic = true;
dates = "19:00";
persistent = true;
options = "--delete-older-than 60d";
};
};
nixpkgs.config.allowUnfree = true ; # Allow Non Free packages
security.sudo.extraConfig = ''
Defaults lecture = never
'';
services = {
cinnamon.apps.enable = false ;
btrbk = {
instances."btrbak" = {
onCalendar = "*-*-* *:00:00";
settings = {
timestamp_format = "long";
snapshot_preserve_min = "2d";
preserve_day_of_week = "sunday" ;
preserve_hour_of_day = "0" ;
target_preserve = "48h 10d 4w 12m 10y" ;
volume."/home" = {
snapshot_create = "always";
subvolume = ".";
snapshot_dir = ".snapshots";
};
volume."/var/local" = {
snapshot_create = "always";
subvolume = ".";
snapshot_dir = ".snapshots";
};
};
openssh = {
enable = true;
settings = {
PermitRootLogin = "yes" ;
};
};
xserver = {
enable = true;
layout = "us";
libinput.enable = true;
displayManager.gdm.enable = true ;
desktopManager.cinnamon.enable = true;
};
};
# Enable sound.
sound.enable = true;
hardware.pulseaudio.enable = true;
system.stateVersion = "23.05";
systemd.services = {
create-swapfile = {
serviceConfig.Type = "oneshot";
wantedBy = [ "swap-swapfile.swap" ];
script = ''
swapfile="/swap/swapfile"
if [ -f "$swapfile" ]; then
echo "Swap file $swapfile already exists, taking no action"
else
echo "Setting up swap file $swapfile"
${pkgs.coreutils}/bin/truncate -s 0 "$swapfile"
${pkgs.e2fsprogs}/bin/chattr +C "$swapfile"
fi
'';
};
};
time.timeZone = "America/Vancouver";
users.users.dave = {
description = "Dave Conroy";
createHome = true ;
home = "/home/dave" ;
shell = "/bin/sh" ; # This is actually bash
group = "users" ;
extraGroups = [ "wheel" "docker" ];
uid = 2323;
isNormalUser = true;
hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
packages = with pkgs; [
];
};
virtualisation = {
docker.enable = true;
};
environment.systemPackages = with pkgs; [
ncdu
kakoune
git
rsync
tmux
fzf
];
environment.persistence."/persist" = {
hideMounts = true ;
directories = [
"/etc/nixos" # NixOS
"/etc/NetworkManager" # NetworkManager
{ directory = "/var/lib/colord"; user = "colord"; group = "colord"; mode = "u=rwx,g=rx,o="; } # Colord Profiles
"/var/lib/bluetooth" # Bluetooth
"/var/lib/docker" # Docker
];
files = [
"/etc/ssh/ssh_host_*" # SSH
"/etc/machine-id" # Needed for SystemD Journal
"/var/lib/NetworkManager/secret_key" # Network Manager
"/var/lib/NetworkManager/seen-bssids" # Network Manager
"/var/lib/NetworkManager/timestamps" # Network Manager
{ file = "/etc/nix/id_rsa"; parentDirectory = { mode = "u=rwx,g=,o="; }; } # Nix
];
};
}
Setup Impermanance
nixos-rebuild boot
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker
nano /etc/nixos/configuration.nix
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id
reboot
Errors
Encrypted /boot
When mounting the ESP partition at /boot/efi instead of /boot grub picks the mounted pool0_0 filesystem and puts it in its configuration.
Idea - Creating a seperate LUKS boot partition??