Skip to main content

Creating LUKS encrypted BTRFS Opt-in Installation with Snapshots

Intro

This details the steps to create a LUKS encrypted BTRFS NixOS installation with impermanance, only keeping the settings that are required in between reboots. This is a work in progress using the standard configuration.nix setup.

Todo

  • Setup in QEMU VM
  • Setup snapper for home snapshots
  • Determine a need for Impermanance module
  • Convert to Flakes
  • Setup on workstation

Setup Variables

DISK_BOOT_SIZE_MB=512
DISK_SWAP_SIZE_GB=4
DISK=/dev/vda

Wipe and Partition Disk

wipefs "${DISK}" -a -f
sgdisk --zap-all "${DISK}"
sgdisk --clear \
                --new=1:0:+"${DISK_BOOT_SIZE_MB}"MiB --typecode=1:ef00 --change-name=1:EFI \
                --new=2:0:+"${DISK_SWAP_SIZE_GB}"GiB --typecode=2:8200 --change-name=2:swap \
                --new=3:0:0 --typecode=3:8300 --change-name=3:pool0_0 \
                "${DISK}"

Create Encrypted Disk

cryptsetup --verify-passphrase -v luksFormat "${DISK}"3
cryptsetup open "${DISK}"3 pool0_0
mkswap "${DISK}"2
swapon "${DISK}"2
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat "${DISK}"1

Create BTRFS Subvolumes

mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log

Take a readonly snapshot of the root subvolume, which will get rolled back to on every boot.

btrfs subvolume snapshot -r /mnt/root /mnt/root-blank

Mount Subvolumes and Partitions

umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt

mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home

mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix

mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist

mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local

mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log

mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot

Generate NixOS Hardware Configuration

nixos-generate-config --root /mnt

Force var_log subvolume to be available for boot

sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix

Force BTRFS mount options

Nixos-generate config isn't smart enough to add options as per here.

sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix

Create Base Configuration

/etc/nixos/configuration.nix

{ config, pkgs, ... }:

{
  imports =
    [ 
      ./hardware-configuration.nix
    ];

  boot = {
    loader = {
      efi = {
        canTouchEfiVariables = false;
      };
      grub = {
        enable = true;
        device = "nodev";  # No device for EFI
        efiSupport = true;
        useOSProber = false;
        efiInstallAsRemovable = true; # in case canTouchEfiVariables doesn't work for your system
      };
    };

    supportedFilesystems = [ 
      "btrfs"
      "fat" "vfat" "exfat" "ntfs" # Microsoft 
      "cifs"                      # Windows Network Share
    ];
  };

  networking = {
    hostName = "beef"; 
    networkmanager.enable = true;
  };

  i18n.defaultLocale = "en_US.UTF-8"; 

  services = {
    cinnamon.apps.enable = false ;
    openssh = {
      enable = true;
      settings = {
        PermitRootLogin = "yes" ;
      };
    };
    xserver = { 
      enable = true;
      layout = "us";
      libinput.enable = true;

      displayManager.gdm.enable = true ;
      desktopManager.cinnamon.enable = true;
    };
  };

  system.stateVersion = "23.05";

  time.timeZone = "America/Vancouver";

  users.users.dave = {
     description = "Dave Conroy";
     createHome = true ;
     home = "/home/dave" ;
     shell = "/bin/sh" ; # This is actually bash
     group = "users" ;
     extraGroups = [ "wheel" "docker" ];
     uid = 2323;
     isNormalUser = true;
     hashedPassword = "$y$j9T$jEYLXjGrR06/tp76fxyDq/$mX4GTWL7CjVXgAcS5nAHEiT6WIH8uD/IfXj16fuTRQ1";
packages = with pkgs; [
     ];
   };

}

Install NixOS

nixos-install

Booting into new system

Configure system and make any changes. Afterwords, run script below to detect changes in snapshot vs root since boot and start working on Persistence mapping.

Script

cat <<EOF > /home/dave/nix-finddiff

#!/usr/bin/env bash

_tmp_root=$(mktemp -d)
mkdir -p "${_tmp_root}"
mount -o subvol=/ /dev/mapper/pool0_0 "${_tmp_root}" > /dev/null 2>&1

set -euo pipefail

OLD_TRANSID=$(sudo btrfs subvolume find-new /mnt/root-blank 9999999)
OLD_TRANSID=${OLD_TRANSID#transid marker was }

sudo btrfs subvolume find-new "/mnt/root" "$OLD_TRANSID" | sed '$d' | cut -f17- -d' ' | sort | uniq | 
while read path; do
    path="/$path"
    if [ -L "$path" ]; then
        : # The path is a symbolic link, so is probably handled by NixOS already
    elif [ -d "$path" ]; then
        : # The path is a directory, ignore
    else
        echo "$path"
    fi
done
umount "${_tmp_root}"
rm -rf "${_tmp_root}"

EOF

Creating Persistence

Bluetooth

  systemd.tmpfiles.rules = [
    "L /var/lib/bluetooth - - - - /persist/var/lib/bluetooth"
  ];

##
sudo mkdir -p /persist/var/lib/bluetooth

Docker

  virtualisation = {
     docker.enable = true;
  };
  
    systemd.tmpfiles.rules = [
    "L /var/lib/docker - - - - /persist/var/lib/docker"
  ];

##
sudo systemctl stop docker
sudo mkdir -p /persist/var/lib/
sudo cp -r {,/persist}/var/lib/docker

Network Manager

  environment.etc."NetworkManager/system-connections" = {
    source = "/persist/etc/NetworkManager/system-connections/";
  };

  systemd.tmpfiles.rules = [
    "L /var/lib/NetworkManager/secret_key - - - - /persist/var/lib/NetworkManager/secret_key"
    "L /var/lib/NetworkManager/seen-bssids - - - - /persist/var/lib/NetworkManager/seen-bssids"
    "L /var/lib/NetworkManager/timestamps - - - - /persist/var/lib/NetworkManager/timestamps"
  ];
  
##
sudo mkdir -p /persist/etc/NetworkManager
sudo cp -r {,/persist}/etc/NetworkManager/system-connections
sudo mkdir -p /persist/var/lib/NetworkManager
sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/

NixOS

 environment.etc = {
    nixos.source = "/persist/etc/nixos";
    NIXOS.source = "/persist/etc/NIXOS";
    machine-id.source = "/persist/etc/machine-id";
  };
    
sudo cp -R {,/persist}/etc/nixos
sudo cp -R {,/persist}/etc/NIXOS
sudo cp {,/persist}/etc/machine-id

OpenSSH

  services.openssh = {
    enable = true;
    hostKeys = [
      {
        path = "/persist/etc/ssh/ssh_host_ed25519_key";
        type = "ed25519";
      }
      {
        path = "/persist/etc/ssh/ssh_host_rsa_key";
        type = "rsa";
        bits = 4096;
      }
    ];
  };

##
sudo mkdir -p /persist/etc/ssh
sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/

Scratchpad

Unmount commands

umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt

Restart Install over and over

umount /boot
umount /mnt/boot
umount /mnt/home
umount /mnt/nix
umount /mnt/persist
umount /mnt/var/local
umount /mnt/var/log
umount /mnt
mkfs.btrfs -f /dev/mapper/pool0_0
mkfs.vfat /dev/vda1
mount -t btrfs /dev/mapper/pool0_0 /mnt
btrfs subvolume create /mnt/root
mkdir -p /mnt/home
btrfs subvolume create /mnt/home/active
btrfs subvolume create /mnt/home/snapshots
btrfs subvolume create /mnt/nix
btrfs subvolume create /mnt/persist
mkdir -p /mnt/var_local
btrfs subvolume create /mnt/var_local/active
btrfs subvolume create /mnt/var_local/snapshots
btrfs subvolume create /mnt/var_log
btrfs subvolume snapshot -r /mnt/root /mnt/root-blank
umount /mnt
mount -o subvol=root,compress=zstd,noatime /dev/mapper/pool0_0 /mnt
mkdir -p /mnt/home
mount -o subvol=home/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/home
mkdir -p /mnt/nix
mount -o subvol=nix,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/nix
mkdir -p /mnt/persist
mount -o subvol=persist,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/persist
mkdir -p /mnt/var/local
mount -o subvol=var_local/active,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/local
mkdir -p /mnt/var/log
mount -o subvol=var_log,compress=zstd,noatime /dev/mapper/pool0_0 /mnt/var/log
mkdir -p /mnt/boot
mount -o defaults,nosuid,nodev,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro /dev/disk/by-partlabel/EFI /mnt/boot
nixos-generate-config --root /mnt
sed -i "/subvol=var_log/a\      neededForBoot = true;" /mnt/etc/nixos/hardware-configuration.nix 
sed -i "s|options = \[ \(.*\) \];|options = \[ \1 \"compress=zstd\" \"noatime\"  \];|g" /mnt/etc/nixos/hardware-configuration.nix 
rm -rf /mnt/etc/nixos/configuration.nix
cp -R /tmp/configuration.nix /mnt/etc/nixos/
nano /mnt/etc/nixos/configuration.nix
nixos-install

   nixos-rebuild boot
   sudo mkdir -p /persist/var/lib/
   sudo cp -r {,/persist}/var/lib/docker
   nano /etc/nixos/configuration.nix 
   sudo mkdir -p /persist/etc/NetworkManager
   sudo cp -r {,/persist}/etc/NetworkManager/system-connections
   sudo mkdir -p /persist/var/lib/NetworkManager
   sudo cp /var/lib/NetworkManager/{secret_key,seen-bssids,timestamps} /persist/var/lib/NetworkManager/  
   sudo mkdir -p /persist/etc/ssh
   sudo cp -R /etc/ssh/ssh_host* /persist/etc/ssh/
   sudo cp -R {,/persist}/etc/nixos
   sudo cp -R {,/persist}/etc/NIXOS
   sudo cp {,/persist}/etc/machine-id
   reboot
Back to top